From 392582bd8184b7c97dae6963e34bb719705592d3 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Sun, 18 Jun 2023 22:22:17 +0100
Subject: [PATCH 01/80] More work on the division of monolithic state protocol
 and divisible state protocol. Still have to actually make the new Log
 Transfer protocol (which will be simple at first, but it could be improved
 for the future.

---
 febft-state-transfer/src/lib.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 30163bd7..a3ea4e61 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -351,12 +351,12 @@ impl<D, OP, NT, PL> StateTransferProtocol<D, OP, NT, PL> for CollabStateTransfer
             CstMessageKind::RequestLatestConsensusSeq => {
                 self.process_request_seq(header, message, order_protocol);
 
-                return Ok(STResult::CstRunning);
+                return Ok(STResult::StateTransferRunning);
             }
             CstMessageKind::RequestState => {
                 self.process_request_state(header, message, order_protocol);
 
-                return Ok(STResult::CstRunning);
+                return Ok(STResult::StateTransferRunning);
             }
             _ => {}
         }
@@ -379,7 +379,7 @@ impl<D, OP, NT, PL> StateTransferProtocol<D, OP, NT, PL> for CollabStateTransfer
 
                 // If we were in the middle of performing a view change, then continue that
                 // View change. If not, then proceed to the normal phase
-                return Ok(STResult::CstFinished(state, requests));
+                return Ok(STResult::StateTransferFinished(state, requests));
             }
             CstStatus::SeqNo(seq) => {
                 if order_protocol.sequence_number() < seq {
@@ -399,7 +399,7 @@ impl<D, OP, NT, PL> StateTransferProtocol<D, OP, NT, PL> for CollabStateTransfer
                     );
                 } else {
                     debug!("{:?} // Not installing sequence number nor requesting state ???? {:?} {:?}", self.node.id(), order_protocol.sequence_number(), seq);
-                    return Ok(STResult::CstNotNeeded);
+                    return Ok(STResult::StateTransferNotNeeded);
                 }
             }
             CstStatus::RequestLatestCid => {
@@ -420,7 +420,7 @@ impl<D, OP, NT, PL> StateTransferProtocol<D, OP, NT, PL> for CollabStateTransfer
             }
         }
 
-        Ok(STResult::CstRunning)
+        Ok(STResult::StateTransferRunning)
     }
 
     fn handle_app_state_requested(&mut self, seq: SeqNo) -> Result<ExecutionResult>

From 453d5968d279a1e479288741563456a6c639f5b1 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 19 Jun 2023 01:04:33 +0100
Subject: [PATCH 02/80] Started working on the new log transfer protocol, which
 will now complement the state transfer protocol, essentially isolating the
 duties of both, which means they can also be run in parallel, instead of
 having to wait for the entire message containing the state and the decision
 log.

---
 febft-state-transfer/src/message/serialize/mod.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/febft-state-transfer/src/message/serialize/mod.rs b/febft-state-transfer/src/message/serialize/mod.rs
index 7c837439..4fe463dd 100644
--- a/febft-state-transfer/src/message/serialize/mod.rs
+++ b/febft-state-transfer/src/message/serialize/mod.rs
@@ -12,6 +12,7 @@ use crate::message::CstMessage;
 pub struct CSTMsg<D: SharedData, OP: OrderingProtocolMessage, SOP: StatefulOrderProtocolMessage>(PhantomData<(D, OP, SOP)>);
 
 impl<D: SharedData, OP: OrderingProtocolMessage, SOP: StatefulOrderProtocolMessage> StateTransferMessage for CSTMsg<D, OP, SOP> {
+
     type StateTransferMessage = CstMessage<D::State, OP::ViewInfo, SOP::DecLog, OP::Proof>;
 
     #[cfg(feature = "serialize_capnp")]

From fe93a38aadadbaa38ae6aaf978e8ee3de7d6514f Mon Sep 17 00:00:00 2001
From: nuno1212s <nuno.neto.g@gmail.com>
Date: Mon, 19 Jun 2023 19:48:24 +0100
Subject: [PATCH 03/80] Almost done with the log transfer protocol. Also almost
 done removing the decision log references from the state transfer protocol.

---
 febft-state-transfer/src/lib.rs               | 532 ++++++++----------
 febft-state-transfer/src/message/mod.rs       |  51 +-
 .../src/message/serialize/mod.rs              |   6 +-
 3 files changed, 265 insertions(+), 324 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index a3ea4e61..358f7676 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -8,6 +8,7 @@ use std::time::Duration;
 use log::{debug, error, info};
 #[cfg(feature = "serialize_serde")]
 use serde::{Deserialize, Serialize};
+use atlas_common::channel::ChannelSyncTx;
 
 use atlas_common::collections;
 use atlas_common::collections::HashMap;
@@ -18,15 +19,17 @@ use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::Node;
 use atlas_communication::message::{Header, NetworkMessageKind, StoredMessage};
-use atlas_execution::app::{Reply, Request, Service, State};
+use atlas_execution::app::{Application, Reply, Request, Service, State};
 use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::SharedData;
 use atlas_core::messages::{StateTransfer, SystemMessage};
 use atlas_core::ordering_protocol::{ExecutionResult, OrderingProtocol, SerProof, View};
-use atlas_core::persistent_log::{PersistableStateTransferProtocol, StateTransferProtocolLog, WriteMode};
-use atlas_core::serialize::{NetworkView, OrderingProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage, StateTransferMessage};
-use atlas_core::state_transfer::{Checkpoint, CstM, DecLog, StatefulOrderProtocol, StateTransferProtocol, STResult, STTimeoutResult};
+use atlas_core::persistent_log::{MonolithicStateLog, PersistableStateTransferProtocol, StateTransferProtocolLog, WriteMode};
+use atlas_core::serialize::{LogTransferMessage, NetworkView, OrderingProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage, StateTransferMessage};
+use atlas_core::state_transfer::{Checkpoint, CstM, StateTransferProtocol, STResult, STTimeoutResult};
+use atlas_core::state_transfer::monolithic_state::MonolithicStateTransfer;
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, Timeouts};
+use atlas_execution::state::monolithic_state::{InstallStateMessage, MonolithicState};
 
 use crate::config::StateTransferConfig;
 use crate::message::{CstMessage, CstMessageKind};
@@ -55,14 +58,14 @@ pub enum CheckpointState<D> {
     Complete(Arc<ReadOnly<Checkpoint<D>>>),
 }
 
-enum ProtoPhase<S, V, O, P> {
+enum ProtoPhase<S> {
     Init,
-    WaitingCheckpoint(Vec<StoredMessage<CstMessage<S, V, O, P>>>),
+    WaitingCheckpoint(Vec<StoredMessage<CstMessage<S>>>),
     ReceivingCid(usize),
     ReceivingState(usize),
 }
 
-impl<S, V, O, P> Debug for ProtoPhase<S, V, O, P> {
+impl<S> Debug for ProtoPhase<S> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
             ProtoPhase::Init => {
@@ -88,79 +91,29 @@ impl<S, V, O, P> Debug for ProtoPhase<S, V, O, P> {
 /// to be very careful thanks to the requests vector, which can be VERY large
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub struct RecoveryState<S, V, O> {
+pub struct RecoveryState<S> {
     pub checkpoint: Arc<ReadOnly<Checkpoint<S>>>,
-    pub view: V,
-    pub log: O,
 }
 
-/// Allow a replica to recover from the state received by peer nodes.
-pub fn install_recovery_state<D, NT, OP, PL>(
-    recovery_state: RecoveryState<D::State, View<OP::Serialization>, DecLog<OP::StateSerialization>>,
-    order_protocol: &mut OP,
-) -> Result<(D::State, Vec<D::Request>)>
-    where
-        D: SharedData + 'static,
-        OP: StatefulOrderProtocol<D, NT, PL>,
-        PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D>
-{
-    // TODO: maybe try to optimize this, to avoid clone(),
-    // which may be quite expensive depending on the size
-    // of the state and the amount of batched requests
-
-    //Because we depend on this state to operate the consensus so it's not that bad that we block
-    //Here, since we're not currently partaking in the consensus.
-    //Also so the executor doesn't have to accept checkpoint types, which is kind of messing with the levels
-    //of the architecture, so I think we can keep it this way
-
-    // TODO: update pub/priv keys when reconfig is implemented?
-
-    let RecoveryState {
-        checkpoint,
-        view,
-        log
-    } = recovery_state;
-
-    let req = order_protocol.install_state(view, log)?;
-
-    let state = checkpoint.state().clone();
-
-    Ok((state, req))
-}
-
-impl<S, O, V> RecoveryState<S, V, O> {
+impl<S> RecoveryState<S> {
     /// Creates a new `RecoveryState`.
     pub fn new(
-        view: V,
         checkpoint: Arc<ReadOnly<Checkpoint<S>>>,
-        declog: O,
     ) -> Self {
         Self {
-            view,
             checkpoint,
-            log: declog,
         }
     }
 
-    /// Returns the view this `RecoveryState` is tracking.
-    pub fn view(&self) -> &V {
-        &self.view
-    }
-
     /// Returns the local checkpoint of this recovery state.
     pub fn checkpoint(&self) -> &Arc<ReadOnly<Checkpoint<S>>> {
         &self.checkpoint
     }
-
-    /// Returns a reference to the decided consensus messages of this recovery state.
-    pub fn log(&self) -> &O {
-        &self.log
-    }
 }
 
-struct ReceivedState<S, V, O> {
+struct ReceivedState<S> {
     count: usize,
-    state: RecoveryState<S, V, O>,
+    state: RecoveryState<S>,
 }
 
 
@@ -170,11 +123,9 @@ struct ReceivedState<S, V, O> {
 ///
 /// The implementation is based on the paper «On the Efficiency of
 /// Durable State Machine Replication», by A. Bessani et al.
-pub struct CollabStateTransfer<D, OP, NT, PL>
-    where D: SharedData + 'static, OP: StatefulOrderProtocol<D, NT, PL> {
-    curr_seq: SeqNo,
-    current_checkpoint_state: CheckpointState<D::State>,
-
+pub struct CollabStateTransfer<S, NT, PL>
+    where S: MonolithicState + 'static {
+    current_checkpoint_state: CheckpointState<S>,
     largest_cid: SeqNo,
     cst_seq: SeqNo,
     latest_cid_count: usize,
@@ -185,14 +136,17 @@ pub struct CollabStateTransfer<D, OP, NT, PL>
     // received already, to avoid replays
     //voted: HashSet<NodeId>,
     node: Arc<NT>,
-    received_states: HashMap<Digest, ReceivedState<D::State, View<OP::Serialization>, DecLog<OP::StateSerialization>>>,
-    phase: ProtoPhase<D::State, View<OP::Serialization>, DecLog<OP::StateSerialization>, SerProof<OP::Serialization>>,
+    received_states: HashMap<Digest, ReceivedState<S>>,
+    phase: ProtoPhase<S>,
+
+    install_channel: ChannelSyncTx<InstallStateMessage<S>>,
 
+    /// Persistent logging for the state transfer protocol.
     persistent_log: PL,
 }
 
 /// Status returned from processing a state transfer message.
-pub enum CstStatus<S, V, O> {
+pub enum CstStatus<S> {
     /// We are not running the CST protocol.
     ///
     /// Drop any attempt of processing a message in this condition.
@@ -200,18 +154,18 @@ pub enum CstStatus<S, V, O> {
     /// The CST protocol is currently running.
     Running,
     /// We should request the latest cid from the view.
-    RequestLatestCid,
-    /// We should request the latest state from the view.
-    RequestState,
-    /// We have received and validated the largest consensus sequence
+    RequestStateCid,
+    /// We have received and validated the largest state sequence
     /// number available.
     SeqNo(SeqNo),
+    /// We should request the latest state from the view.
+    RequestState,
     /// We have received and validated the state from
     /// a group of replicas.
-    State(RecoveryState<S, V, O>),
+    State(RecoveryState<S>),
 }
 
-impl<S, V, O> Debug for CstStatus<S, V, O> {
+impl<S> Debug for CstStatus<S> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
             CstStatus::Nil => {
@@ -220,7 +174,7 @@ impl<S, V, O> Debug for CstStatus<S, V, O> {
             CstStatus::Running => {
                 write!(f, "Running")
             }
-            CstStatus::RequestLatestCid => {
+            CstStatus::RequestStateCid => {
                 write!(f, "Request latest CID")
             }
             CstStatus::RequestState => {
@@ -240,13 +194,13 @@ impl<S, V, O> Debug for CstStatus<S, V, O> {
 ///
 /// To clarify, the mention of state machine here has nothing to do with the
 /// SMR protocol, but rather the implementation in code of the CST protocol.
-pub enum CstProgress<S, V, O, P> {
+pub enum CstProgress<S> {
     // TODO: Timeout( some type here)
     /// This value represents null progress in the CST code's state machine.
     Nil,
     /// We have a fresh new message to feed the CST state machine, from
     /// the communication layer.
-    Message(Header, CstMessage<S, V, O, P>),
+    Message(Header, CstMessage<S>),
 }
 
 macro_rules! getmessage {
@@ -266,53 +220,45 @@ macro_rules! getmessage {
     }};
 }
 
-type Serialization<S, D, OP, NT, PL> = <S as StateTransferProtocol<D, OP, NT, PL>>::Serialization;
+type Serialization<S, OP, NT> = <S as StateTransferProtocol<S, OP, NT>>::Serialization;
 
-impl<D, OP, NT, PL> StateTransferProtocol<D, OP, NT, PL> for CollabStateTransfer<D, OP, NT, PL>
-    where D: SharedData + 'static,
-          OP: StatefulOrderProtocol<D, NT, PL> + 'static
+impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT, PL>
+    where S: MonolithicState + 'static,
 {
-    type Serialization = CSTMsg<D, OP::Serialization, OP::StateSerialization>;
-    type Config = StateTransferConfig;
+    type Serialization = CSTMsg<S>;
 
-    fn initialize(config: Self::Config, timeouts: Timeouts, node: Arc<NT>, persistent_log: PL)
-                  -> Result<Self>
-        where
-            Self: Sized {
-        let StateTransferConfig {
-            timeout_duration
-        } = config;
-
-        Ok(CollabStateTransfer::<D, OP, NT, PL>::new(node, timeout_duration, timeouts, persistent_log))
-    }
-
-    fn request_latest_state(&mut self, order_protocol: &mut OP) -> Result<()>
-        where NT: Node<ServiceMsg<D, OP::Serialization, Self::Serialization>>,
-              PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D> {
-        self.request_latest_consensus_seq_no(order_protocol);
+    fn request_latest_state<D, OP, LP, V>(&mut self, view: V) -> Result<()>
+        where D: SharedData + 'static,
+              OP: OrderingProtocolMessage,
+              LP: LogTransferMessage,
+              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              V: NetworkView {
+        self.request_latest_consensus_seq_no::<D, OP, LP, V>(view);
 
         Ok(())
     }
 
-    fn handle_off_ctx_message(&mut self, order_protocol: &mut OP,
-                              message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
-                              -> Result<()>
-        where NT: Node<ServiceMsg<D, OP::Serialization, Self::Serialization>>,
-              PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D> {
+    fn handle_off_ctx_message<D, OP, LP, V>(&mut self, view: V, message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
+                                            -> Result<()>
+        where D: SharedData + 'static,
+              OP: OrderingProtocolMessage,
+              LP: LogTransferMessage,
+              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              V: NetworkView {
         let (header, message) = message.into_inner();
 
         let message = message.into_inner();
-        debug!("{:?} // Off context Message {:?} from {:?} with seq {:?}", self.node.id(),
-            message, header.from(), message.sequence_number());
+
+        debug!("{:?} // Off context Message {:?} from {:?} with seq {:?}", self.node.id(), message, header.from(), message.sequence_number());
 
         match &message.kind() {
             CstMessageKind::RequestLatestConsensusSeq => {
-                self.process_request_seq(header, message, order_protocol);
+                self.process_request_seq(header, message);
 
                 return Ok(());
             }
             CstMessageKind::RequestState => {
-                self.process_request_state(header, message, order_protocol);
+                self.process_request_state(header, message);
 
                 return Ok(());
             }
@@ -320,8 +266,8 @@ impl<D, OP, NT, PL> StateTransferProtocol<D, OP, NT, PL> for CollabStateTransfer
         }
 
         let status = self.process_message(
+            view,
             CstProgress::Message(header, message),
-            order_protocol,
         );
 
         match status {
@@ -335,82 +281,63 @@ impl<D, OP, NT, PL> StateTransferProtocol<D, OP, NT, PL> for CollabStateTransfer
         Ok(())
     }
 
-    fn process_message(&mut self, order_protocol: &mut OP,
-                       message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
-                       -> Result<STResult<D>>
-        where NT: Node<ServiceMsg<D, OP::Serialization, Self::Serialization>>,
-              PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D> {
+    fn process_message<D, OP, LP, V>(&mut self,
+                                     view: V,
+                                     message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
+                                     -> Result<STResult>
+        where D: SharedData + 'static,
+              OP: OrderingProtocolMessage,
+              LP: LogTransferMessage,
+              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              V: NetworkView {
         let (header, message) = message.into_inner();
 
         let message = message.into_inner();
 
-        debug!("{:?} // Message {:?} from {:?} while in phase {:?}", self.node.id(),
-            message, header.from(), self.phase);
+        debug!("{:?} // Message {:?} from {:?} while in phase {:?}", self.node.id(), message, header.from(), self.phase);
 
         match &message.kind() {
             CstMessageKind::RequestLatestConsensusSeq => {
-                self.process_request_seq(header, message, order_protocol);
+                self.process_request_seq(header, message);
 
                 return Ok(STResult::StateTransferRunning);
             }
             CstMessageKind::RequestState => {
-                self.process_request_state(header, message, order_protocol);
+                self.process_request_state(header, message);
 
                 return Ok(STResult::StateTransferRunning);
             }
             _ => {}
         }
 
-        // Notify timeouts that we have received this message
+// Notify timeouts that we have received this message
         self.timeouts.received_cst_request(header.from(), message.sequence_number());
 
-        let status = self.process_message(
-            CstProgress::Message(header, message),
-            order_protocol,
-        );
+        let status = self.process_message(view.clone(),
+                                          CstProgress::Message(header, message), );
 
         match status {
             CstStatus::Running => (),
             CstStatus::State(state) => {
-                let (state, requests) = install_recovery_state(
-                    state,
-                    order_protocol,
-                )?;
-
-                // If we were in the middle of performing a view change, then continue that
-                // View change. If not, then proceed to the normal phase
-                return Ok(STResult::StateTransferFinished(state, requests));
+                self.install_channel.send(InstallStateMessage::new(state.checkpoint.clone())).unwrap();
+
+                return Ok(STResult::StateTransferFinished(state.checkpoint.sequence_number()));
             }
             CstStatus::SeqNo(seq) => {
-                if order_protocol.sequence_number() < seq {
-                    debug!("{:?} // Installing sequence number and requesting state {:?}", self.node.id(), seq);
+                if self.current_checkpoint_state.sequence_number() < seq {
+                    debug!("{:?} // Requesting state {:?}", self.node.id(), seq);
 
-                    // this step will allow us to ignore any messages
-                    // for older consensus instances we may have had stored;
-                    //
-                    // after we receive the latest recovery state, we
-                    // need to install the then latest sequence no;
-                    // this is done with the function
-                    // `install_recovery_state` from cst
-                    order_protocol.install_seq_no(seq)?;
-
-                    self.request_latest_state(
-                        order_protocol,
-                    );
+                    self.request_latest_state(view);
                 } else {
                     debug!("{:?} // Not installing sequence number nor requesting state ???? {:?} {:?}", self.node.id(), order_protocol.sequence_number(), seq);
-                    return Ok(STResult::StateTransferNotNeeded);
+                    return Ok(STResult::StateTransferNotNeeded(seq));
                 }
             }
-            CstStatus::RequestLatestCid => {
-                self.request_latest_consensus_seq_no(
-                    order_protocol,
-                );
+            CstStatus::RequestStateCid => {
+                self.request_latest_consensus_seq_no(view);
             }
             CstStatus::RequestState => {
-                self.request_latest_state(
-                    order_protocol,
-                );
+                self.request_latest_state(view);
             }
             CstStatus::Nil => {
                 // No actions are required for the CST
@@ -423,9 +350,11 @@ impl<D, OP, NT, PL> StateTransferProtocol<D, OP, NT, PL> for CollabStateTransfer
         Ok(STResult::StateTransferRunning)
     }
 
-    fn handle_app_state_requested(&mut self, seq: SeqNo) -> Result<ExecutionResult>
-        where NT: Node<ServiceMsg<D, OP::Serialization, Self::Serialization>>,
-              PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D> {
+    fn handle_app_state_requested<D, OP, LP, V>(&mut self, view: V, seq: SeqNo) -> Result<ExecutionResult>
+        where D: SharedData + 'static,
+              OP: OrderingProtocolMessage,
+              LP: LogTransferMessage,
+              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>, V: NetworkView {
         let earlier = std::mem::replace(&mut self.current_checkpoint_state, CheckpointState::None);
 
         self.current_checkpoint_state = match earlier {
@@ -448,16 +377,51 @@ impl<D, OP, NT, PL> StateTransferProtocol<D, OP, NT, PL> for CollabStateTransfer
         Ok(ExecutionResult::BeginCheckpoint)
     }
 
-    fn handle_state_received_from_app(&mut self, order_protocol: &mut OP, state: Arc<ReadOnly<Checkpoint<D::State>>>) -> Result<()>
-        where NT: Node<ServiceMsg<D, OP::Serialization, Self::Serialization>>,
-              PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D> {
-        order_protocol.checkpointed(state.sequence_number())?;
+    fn handle_timeout<D, OP, LP, V>(&mut self, view: V, timeout: Vec<RqTimeout>) -> Result<STTimeoutResult>
+        where D: SharedData + 'static,
+              OP: OrderingProtocolMessage,
+              LP: LogTransferMessage,
+              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              V: NetworkView {
+        for cst_seq in timeout {
+            if let TimeoutKind::Cst(cst_seq) = cst_seq.timeout_kind() {
+                if self.cst_request_timed_out(cst_seq.clone(), view.clone()) {
+                    return Ok(STTimeoutResult::RunCst);
+                }
+            }
+        }
+
+        Ok(STTimeoutResult::CstNotNeeded)
+    }
+}
+
+impl<S, NT, PL> MonolithicStateTransfer<S, NT, PL> for CollabStateTransfer<S, NT, PL>
+    where S: MonolithicState + 'static,
+          PL: MonolithicStateLog<S> {
+    type Config = StateTransferConfig;
+
+    fn initialize(config: Self::Config, timeouts: Timeouts, node: Arc<NT>,
+                  log: PL, executor_handle: ChannelSyncTx<InstallStateMessage<S>>) -> Result<Self>
+        where Self: Sized {
+        let StateTransferConfig {
+            timeout_duration
+        } = config;
+
 
+        Ok(Self::new(node, timeout_duration, timeouts, log, executor_handle))
+    }
+
+    fn handle_state_received_from_app<D, OP, LP, V>(&mut self, view: V, state: Arc<ReadOnly<Checkpoint<S>>>) -> Result<()>
+        where D: SharedData + 'static,
+              OP: OrderingProtocolMessage,
+              LP: LogTransferMessage,
+              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              V: NetworkView {
         self.finalize_checkpoint(state)?;
 
         if self.needs_checkpoint() {
             // This will make the state transfer protocol aware of the latest state
-            if let CstStatus::Nil = self.process_message(CstProgress::Nil, order_protocol) {} else {
+            if let CstStatus::Nil = self.process_message(view, CstProgress::Nil) {} else {
                 return Err("Process message while needing checkpoint returned something else than nil")
                     .wrapped(ErrorKind::Cst);
             }
@@ -465,33 +429,18 @@ impl<D, OP, NT, PL> StateTransferProtocol<D, OP, NT, PL> for CollabStateTransfer
 
         Ok(())
     }
-
-    fn handle_timeout(&mut self, order_protocol: &mut OP, timeout: Vec<RqTimeout>) -> Result<STTimeoutResult>
-        where NT: Node<ServiceMsg<D, OP::Serialization, Self::Serialization>>,
-              PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D> {
-        for cst_seq in timeout {
-            if let TimeoutKind::Cst(cst_seq) = cst_seq.timeout_kind() {
-                if self.cst_request_timed_out(cst_seq.clone(), order_protocol) {
-                    return Ok(STTimeoutResult::RunCst);
-                }
-            }
-        }
-
-        Ok(STTimeoutResult::CstNotNeeded)
-    }
 }
 
 // TODO: request timeouts
-impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
+impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     where
-        D: SharedData + 'static,
-        OP: StatefulOrderProtocol<D, NT, PL>
+        S: MonolithicState + 'static,
+        PL: MonolithicStateLog<S>,
 {
     /// Create a new instance of `CollabStateTransfer`.
-    pub fn new(node: Arc<NT>, base_timeout: Duration, timeouts: Timeouts, persistent_log: PL) -> Self {
+    pub fn new(node: Arc<NT>, base_timeout: Duration, timeouts: Timeouts, persistent_log: PL, install_channel: ChannelSyncTx<InstallStateMessage<S>>) -> Self {
         Self {
             current_checkpoint_state: CheckpointState::None,
-            curr_seq: SeqNo::ZERO,
             base_timeout,
             curr_timeout: base_timeout,
             timeouts,
@@ -502,6 +451,7 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
             latest_cid_count: 0,
             cst_seq: SeqNo::ZERO,
             persistent_log,
+            install_channel,
         }
     }
 
@@ -513,27 +463,28 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
         matches!(self.phase, ProtoPhase::WaitingCheckpoint(_))
     }
 
-    fn process_request_seq(
+    fn process_request_seq<D, OP, LP>(
         &mut self,
         header: Header,
-        message: CstMessage<D::State, View<OP::Serialization>, DecLog<OP::StateSerialization>, SerProof<OP::Serialization>>,
-        order_protocol: &mut OP, )
-        where
-            OP: StatefulOrderProtocol<D, NT, PL>,
-            NT: Node<ServiceMsg<D, OP::Serialization, CSTMsg<D, OP::Serialization, OP::StateSerialization>>>,
-            PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D>
+        message: CstMessage<S>)
+        where D: SharedData + 'static,
+              OP: OrderingProtocolMessage,
+              LP: LogTransferMessage,
+              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>
     {
-        let proof = match order_protocol.sequence_number_with_proof() {
-            Ok(res) => {
-                res
+        let seq = match &self.current_checkpoint_state {
+            CheckpointState::PartialWithEarlier { seq, earlier, } => {
+                Some((earlier.sequence_number(), earlier.digest().clone()))
             }
-            Err(err) => {
-                error!("{:?} // Error while getting sequence number with proof {:?}", self.node.id(), err);
-                return;
+            CheckpointState::Complete(seq) => {
+                Some((seq.sequence_number(), seq.digest().clone()))
+            }
+            _ => {
+                None
             }
         };
 
-        let kind = CstMessageKind::ReplyLatestConsensusSeq(proof);
+        let kind = CstMessageKind::ReplyStateCid(seq);
 
         let reply = CstMessage::new(message.sequence_number(), kind);
 
@@ -548,15 +499,15 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
 
     /// Process the entire list of pending state transfer requests
     /// This will only reply to the latest request sent by each of the replicas
-    fn process_pending_state_requests(&mut self, order_protocol: &mut OP)
-        where
-            OP: StatefulOrderProtocol<D, NT, PL>,
-            NT: Node<ServiceMsg<D, OP::Serialization, CSTMsg<D, OP::Serialization, OP::StateSerialization>>>,
-            PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D> {
+    fn process_pending_state_requests<D, OP, LP>(&mut self)
+        where D: SharedData + 'static,
+              OP: OrderingProtocolMessage,
+              LP: LogTransferMessage,
+              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>> {
         let waiting = std::mem::replace(&mut self.phase, ProtoPhase::Init);
 
         if let ProtoPhase::WaitingCheckpoint(reqs) = waiting {
-            let mut map: HashMap<NodeId, StoredMessage<CstMessage<D::State, View<OP::Serialization>, DecLog<OP::StateSerialization>, SerProof<OP::Serialization>>>> = collections::hash_map();
+            let mut map: HashMap<NodeId, StoredMessage<CstMessage<S>>> = collections::hash_map();
 
             for request in reqs {
                 // We only want to reply to the most recent requests from each of the nodes
@@ -577,20 +528,19 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
             map.into_values().for_each(|req| {
                 let (header, message) = req.into_inner();
 
-                self.process_request_state(header, message, order_protocol);
+                self.process_request_state(header, message);
             });
         }
     }
 
-    fn process_request_state(
+    fn process_request_state<D, OP, LP, V>(
         &mut self,
         header: Header,
-        message: CstMessage<D::State, View<OP::Serialization>, DecLog<OP::StateSerialization>, SerProof<OP::Serialization>>,
-        op: &mut OP,
-    ) where
-        OP: StatefulOrderProtocol<D, NT, PL>,
-        NT: Node<ServiceMsg<D, OP::Serialization, CSTMsg<D, OP::Serialization, OP::StateSerialization>>>,
-        PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D>
+        message: CstMessage<S>,
+    ) where D: SharedData + 'static,
+            OP: OrderingProtocolMessage,
+            LP: LogTransferMessage,
+            NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>
     {
         match &mut self.phase {
             ProtoPhase::Init => {}
@@ -621,48 +571,33 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
             }
         };
 
-        let (view, dec_log) = match op.snapshot_log() {
-            Ok((view, dec_log)) => { (view, dec_log) }
-            Err(_) => {
-                if let ProtoPhase::WaitingCheckpoint(waiting) = &mut self.phase {
-                    waiting.push(StoredMessage::new(header, message));
-                } else {
-                    self.phase = ProtoPhase::WaitingCheckpoint(vec![StoredMessage::new(header, message)]);
-                }
-
-                return;
-            }
-        };
-
         let reply = CstMessage::new(
             message.sequence_number(),
             CstMessageKind::ReplyState(RecoveryState {
                 checkpoint: state,
-                view,
-                log: dec_log,
             }),
         );
 
         let network_msg = NetworkMessageKind::from(SystemMessage::from_state_transfer_message(reply));
 
-        self.node.send(network_msg,
-                       header.from(), true).unwrap();
+        self.node.send(network_msg, header.from(), true).unwrap();
     }
 
     /// Advances the state of the CST state machine.
-    pub fn process_message(
+    pub fn process_message<D, OP, LP, V>(
         &mut self,
-        progress: CstProgress<D::State, View<OP::Serialization>, DecLog<OP::StateSerialization>, SerProof<OP::Serialization>>,
-        order_protocol: &mut OP,
-    ) -> CstStatus<D::State, View<OP::Serialization>, DecLog<OP::StateSerialization>>
-        where
-            OP: StatefulOrderProtocol<D, NT, PL>,
-            NT: Node<ServiceMsg<D, OP::Serialization, CSTMsg<D, OP::Serialization, OP::StateSerialization>>>,
-            PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D>
+        view: V,
+        progress: CstProgress<S>,
+    ) -> CstStatus<S>
+        where D: SharedData + 'static,
+              OP: OrderingProtocolMessage,
+              LP: LogTransferMessage,
+              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              V: NetworkView
     {
         match self.phase {
             ProtoPhase::WaitingCheckpoint(_) => {
-                self.process_pending_state_requests(order_protocol);
+                self.process_pending_state_requests();
 
                 CstStatus::Nil
             }
@@ -671,10 +606,10 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
 
                 match message.kind() {
                     CstMessageKind::RequestLatestConsensusSeq => {
-                        self.process_request_seq(header, message, order_protocol);
+                        self.process_request_seq(header, message);
                     }
                     CstMessageKind::RequestState => {
-                        self.process_request_state(header, message, order_protocol);
+                        self.process_request_state(header, message);
                     }
                     // we are not running cst, so drop any reply msgs
                     //
@@ -687,7 +622,7 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
                 CstStatus::Nil
             }
             ProtoPhase::ReceivingCid(i) => {
-                let (header, message) = getmessage!(progress, CstStatus::RequestLatestCid);
+                let (header, message) = getmessage!(progress, CstStatus::RequestStateCid);
 
                 debug!("{:?} // Received Cid with {} responses from {:?} for CST Seq {:?} vs Ours {:?}", self.node.id(),
                    i, header.from(), message.sequence_number(), self.cst_seq);
@@ -706,20 +641,8 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
                 }
 
                 match message.kind() {
-                    CstMessageKind::ReplyLatestConsensusSeq(proof) => {
-                        let seq = if let Some((seq, proof)) = proof {
-                            if let Ok(verified) = order_protocol.verify_sequence_number(*seq, proof) {
-                                if verified {
-                                    *seq
-                                } else {
-                                    SeqNo::ZERO
-                                }
-                            } else {
-                                SeqNo::ZERO
-                            }
-                        } else {
-                            SeqNo::ZERO
-                        };
+                    CstMessageKind::ReplyStateCid(state_cid) => {
+                        todo!();
 
                         debug!("{:?} // Received CID vote {:?} from {:?}", self.node.id(), seq, header.from());
 
@@ -735,12 +658,12 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
                         }
                     }
                     CstMessageKind::RequestLatestConsensusSeq => {
-                        self.process_request_seq(header, message, order_protocol);
+                        self.process_request_seq(header, message);
 
                         return CstStatus::Running;
                     }
                     CstMessageKind::RequestState => {
-                        self.process_request_state(header, message, order_protocol);
+                        self.process_request_state(header, message);
 
                         return CstStatus::Running;
                     }
@@ -755,10 +678,10 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
                 let i = i + 1;
 
                 debug!("{:?} // Quorum count {}, i: {}, cst_seq {:?}. Current Latest Cid: {:?}. Current Latest Cid Count: {}",
-                        self.node.id(), order_protocol.view().quorum(), i,
+                        self.node.id(), view.quorum(), i,
                         self.cst_seq, self.largest_cid, self.latest_cid_count);
 
-                if i == order_protocol.view().quorum() {
+                if i == view.quorum() {
                     self.phase = ProtoPhase::Init;
 
                     // reset timeout, since req was successful
@@ -828,7 +751,7 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
                 // TODO: check for more than one reply from the same node
                 let i = i + 1;
 
-                if i <= order_protocol.view().f() {
+                if i <= view.f() {
                     self.phase = ProtoPhase::ReceivingState(i);
                     return CstStatus::Running;
                 }
@@ -844,7 +767,7 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
                     match received_state {
                         Some((digest, _)) => digest.clone(),
                         None => {
-                            return if i >= order_protocol.view().quorum() {
+                            return if i >= view.quorum() {
                                 self.received_states.clear();
 
                                 debug!("{:?} // No matching states found, clearing", self.node.id());
@@ -866,7 +789,7 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
                 self.curr_timeout = self.base_timeout;
 
                 // return the state
-                let f = order_protocol.view().f();
+                let f = view.f();
 
                 match received_state {
                     Some(ReceivedState { count, state }) if count > f => {
@@ -890,8 +813,8 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
     /// This method should only be called when `finalize_request()` reports
     /// `Info::BeginCheckpoint`, and the requested application state is received
     /// on the core server task's master channel.
-    pub fn finalize_checkpoint(&mut self, checkpoint: Arc<ReadOnly<Checkpoint<D::State>>>) -> Result<()> where
-        PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D> {
+    pub fn finalize_checkpoint(&mut self, checkpoint: Arc<ReadOnly<Checkpoint<S>>>) -> Result<()> where
+        PL: MonolithicStateLog<S> {
         match &self.current_checkpoint_state {
             CheckpointState::None => {
                 Err("No checkpoint has been initiated yet").wrapped(ErrorKind::MsgLog)
@@ -924,26 +847,21 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
     /// Handle a timeout received from the timeouts layer.
     /// Returns a bool to signify if we must move to the Retrieving state
     /// If the timeout is no longer relevant, returns false (Can remain in current phase)
-    pub fn cst_request_timed_out(&mut self, seq: SeqNo,
-                                 order_protocol: &OP) -> bool
-        where
-            OP: StatefulOrderProtocol<D, NT, PL> + 'static,
-            NT: Node<ServiceMsg<D, OP::Serialization, Serialization<Self, D, OP, NT, PL>>>,
-            PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D> {
+    pub fn cst_request_timed_out<D, OP, LP, V>(&mut self, seq: SeqNo, view: V) -> bool
+        where D: SharedData + 'static,
+              OP: OrderingProtocolMessage,
+              LP: LogTransferMessage,
+              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>, V: NetworkView {
         let status = self.timed_out(seq);
 
         match status {
-            CstStatus::RequestLatestCid => {
-                self.request_latest_consensus_seq_no(
-                    order_protocol,
-                );
+            CstStatus::RequestStateCid => {
+                self.request_latest_consensus_seq_no(view);
 
                 true
             }
             CstStatus::RequestState => {
-                self.request_latest_state(
-                    order_protocol,
-                );
+                self.request_latest_state(view);
 
                 true
             }
@@ -952,7 +870,7 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
         }
     }
 
-    fn timed_out(&mut self, seq: SeqNo) -> CstStatus<D::State, View<OP::Serialization>, DecLog<OP::StateSerialization>> {
+    fn timed_out(&mut self, seq: SeqNo) -> CstStatus<S> {
         if seq != self.cst_seq {
             // the timeout we received is for a request
             // that has already completed, therefore we ignore it
@@ -970,7 +888,7 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
             // retry requests if receiving state and we have timed out
             ProtoPhase::ReceivingCid(_) => {
                 self.curr_timeout *= 2;
-                CstStatus::RequestLatestCid
+                CstStatus::RequestStateCid
             }
             ProtoPhase::ReceivingState(_) => {
                 self.curr_timeout *= 2;
@@ -984,13 +902,14 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
 
     /// Used by a recovering node to retrieve the latest sequence number
     /// attributed to a client request by the consensus layer.
-    pub fn request_latest_consensus_seq_no(
+    pub fn request_latest_consensus_seq_no<D, OP, LP, V>(
         &mut self,
-        order_protocol: &OP,
-    ) where
-        OP: StatefulOrderProtocol<D, NT, PL> + 'static,
-        NT: Node<ServiceMsg<D, OP::Serialization, Serialization<Self, D, OP, NT, PL>>>,
-        PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D>
+        view: V,
+    ) where D: SharedData + 'static,
+            OP: OrderingProtocolMessage,
+            LP: LogTransferMessage,
+            NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+            V: NetworkView
     {
 
         // reset state of latest seq no. request
@@ -998,12 +917,11 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
         self.latest_cid_count = 0;
 
         let cst_seq = self.curr_seq();
-        let current_view = order_protocol.view();
 
         info!("{:?} // Requesting latest consensus seq no with seq {:?}", self.node.id(), cst_seq);
 
         self.timeouts.timeout_cst_request(self.curr_timeout,
-                                          current_view.quorum() as u32,
+                                          view.quorum() as u32,
                                           cst_seq);
 
         self.phase = ProtoPhase::ReceivingCid(0);
@@ -1013,30 +931,29 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
             CstMessageKind::RequestLatestConsensusSeq,
         );
 
-        let targets = NodeId::targets(0..current_view.n());
+        let targets = NodeId::targets(0..view.n());
 
         self.node.broadcast(NetworkMessageKind::from(SystemMessage::from_state_transfer_message(message)), targets);
     }
 
     /// Used by a recovering node to retrieve the latest state.
-    pub fn request_latest_state(
-        &mut self,
-        order_protocol: &OP,
-    ) where
-        OP: StatefulOrderProtocol<D, NT, PL> + 'static,
-        NT: Node<ServiceMsg<D, OP::Serialization, CSTMsg<D, OP::Serialization, OP::StateSerialization>>>,
-        PL: StateTransferProtocolLog<OP::Serialization, OP::StateSerialization, D>
+    pub fn request_latest_state<D, OP, LP, V>(
+        &mut self, view: V,
+    ) where D: SharedData + 'static,
+            OP: OrderingProtocolMessage,
+            LP: LogTransferMessage,
+            NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+            V: NetworkView
     {
         // reset hashmap of received states
         self.received_states.clear();
 
         let cst_seq = self.curr_seq();
-        let current_view = order_protocol.view();
 
         info!("{:?} // Requesting latest state with cst msg seq {:?}", self.node.id(), cst_seq);
 
         self.timeouts.timeout_cst_request(self.curr_timeout,
-                                          current_view.quorum() as u32,
+                                          view.quorum() as u32,
                                           cst_seq);
 
         self.phase = ProtoPhase::ReceivingState(0);
@@ -1044,14 +961,31 @@ impl<D, OP, NT, PL> CollabStateTransfer<D, OP, NT, PL>
         //TODO: Maybe attempt to use followers to rebuild state and avoid
         // Overloading the replicas
         let message = CstMessage::new(cst_seq, CstMessageKind::RequestState);
-        let targets = NodeId::targets(0..current_view.n()).filter(|id| *id != self.node.id());
+        let targets = NodeId::targets(0..view.n()).filter(|id| *id != self.node.id());
 
         self.node.broadcast(NetworkMessageKind::from(SystemMessage::from_state_transfer_message(message)), targets);
     }
 }
 
-impl<D, OP, NT, PL> PersistableStateTransferProtocol for CollabStateTransfer<D, OP, NT, PL>
-    where
-        D: SharedData + 'static,
-        OP: StatefulOrderProtocol<D, NT, PL>,
-        NT: Send, PL: Send {}
+impl<S, NT, PL> PersistableStateTransferProtocol for CollabStateTransfer<S, NT, PL>
+    where S: MonolithicState + 'static {}
+
+
+impl<S> Orderable for CheckpointState<S> {
+    fn sequence_number(&self) -> SeqNo {
+        match self {
+            CheckpointState::None => {
+                SeqNo::ZERO
+            }
+            CheckpointState::Partial { seq } => {
+                SeqNo::ZERO
+            }
+            CheckpointState::PartialWithEarlier { earlier, .. } => {
+                earlier.sequence_number()
+            }
+            CheckpointState::Complete(arc) => {
+                arc.sequence_number()
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/febft-state-transfer/src/message/mod.rs b/febft-state-transfer/src/message/mod.rs
index 20b7c0a4..b95d45cc 100644
--- a/febft-state-transfer/src/message/mod.rs
+++ b/febft-state-transfer/src/message/mod.rs
@@ -1,71 +1,78 @@
-pub mod serialize;
-
-
 use std::fmt::{Debug, Formatter};
+
 #[cfg(feature = "serialize_serde")]
 use serde::{Deserialize, Serialize};
+use atlas_common::crypto::hash::Digest;
+
 use atlas_common::ordering::{Orderable, SeqNo};
+
 use crate::RecoveryState;
 
+pub mod serialize;
+
+
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub struct CstMessage<S, V, O, P> {
+pub struct CstMessage<S> {
     // NOTE: not the same sequence number used in the
     // consensus layer to order client requests!
     seq: SeqNo,
-    kind: CstMessageKind<S, V, O, P>,
+    kind: CstMessageKind<S>,
 }
 
-impl<S, V, O, P> Debug for CstMessage<S, V, O, P> {
+impl<S> Debug for CstMessage<S> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match &self.kind {
-            CstMessageKind::RequestLatestConsensusSeq => {
-                write!(f, "Request consensus ID")
-            }
-            CstMessageKind::ReplyLatestConsensusSeq(opt ) => {
-                write!(f, "Reply consensus seq {:?}", opt.as_ref().map(|(seq, _)| *seq).unwrap_or(SeqNo::ZERO))
-            }
             CstMessageKind::RequestState => {
                 write!(f, "Request state message")
             }
             CstMessageKind::ReplyState(_) => {
                 write!(f, "Reply with state message")
             }
+            CstMessageKind::RequestStateCid => {
+                write!(f, "Request state cid message")
+            }
+            CstMessageKind::ReplyStateCid(opt) => {
+                if let Some((seq, digest)) = opt {
+                    write!(f, "Reply with state cid message {:?} {:?}", seq, digest)
+                } else {
+                    write!(f, "Reply with state cid message None")
+                }
+            }
         }
-
     }
 }
 
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub enum CstMessageKind<S, V, O, P> {
-    RequestLatestConsensusSeq,
-    ReplyLatestConsensusSeq(Option<(SeqNo, P)>),
+pub enum CstMessageKind<S> {
+    RequestStateCid,
+    ReplyStateCid(Option<(SeqNo, Digest)>),
     RequestState,
-    ReplyState(RecoveryState<S, V, O>),
+    ReplyState(RecoveryState<S>),
 }
 
-impl<S, O, V, P> Orderable for CstMessage<S, V, O, P> {
+impl<S> Orderable for CstMessage<S> {
     /// Returns the sequence number of this state transfer message.
     fn sequence_number(&self) -> SeqNo {
         self.seq
     }
 }
 
-impl<S, O, V, P> CstMessage<S, V, O, P> {
+impl<S> CstMessage<S> {
     /// Creates a new `CstMessage` with sequence number `seq`,
     /// and of the kind `kind`.
-    pub fn new(seq: SeqNo, kind: CstMessageKind<S, V, O, P>) -> Self {
+    pub fn new(seq: SeqNo, kind: CstMessageKind<S>) -> Self {
         Self { seq, kind }
     }
 
     /// Returns a reference to the state transfer message kind.
-    pub fn kind(&self) -> &CstMessageKind<S, V, O, P> {
+    pub fn kind(&self) -> &CstMessageKind<S> {
         &self.kind
     }
 
     /// Takes the recovery state embedded in this cst message, if it is available.
-    pub fn take_state(&mut self) -> Option<RecoveryState<S, V, O>> {
+    pub fn take_state(&mut self) -> Option<RecoveryState<S>> {
         let kind = std::mem::replace(&mut self.kind, CstMessageKind::RequestState);
         match kind {
             CstMessageKind::ReplyState(state) => Some(state),
diff --git a/febft-state-transfer/src/message/serialize/mod.rs b/febft-state-transfer/src/message/serialize/mod.rs
index 4fe463dd..542e0536 100644
--- a/febft-state-transfer/src/message/serialize/mod.rs
+++ b/febft-state-transfer/src/message/serialize/mod.rs
@@ -9,11 +9,11 @@ use atlas_core::state_transfer::{StatefulOrderProtocol, StateTransferProtocol};
 use crate::CollabStateTransfer;
 use crate::message::CstMessage;
 
-pub struct CSTMsg<D: SharedData, OP: OrderingProtocolMessage, SOP: StatefulOrderProtocolMessage>(PhantomData<(D, OP, SOP)>);
+pub struct CSTMsg<S>(PhantomData<(S)>);
 
-impl<D: SharedData, OP: OrderingProtocolMessage, SOP: StatefulOrderProtocolMessage> StateTransferMessage for CSTMsg<D, OP, SOP> {
+impl<S> StateTransferMessage for CSTMsg<S> {
 
-    type StateTransferMessage = CstMessage<D::State, OP::ViewInfo, SOP::DecLog, OP::Proof>;
+    type StateTransferMessage = CstMessage<S>;
 
     #[cfg(feature = "serialize_capnp")]
     fn serialize_capnp(builder: atlas_capnp::cst_messages_capnp::cst_message::Builder, msg: &Self::StateTransferMessage) -> atlas_common::error::Result<()> {

From 827620157f6bec42a43a5f1fe7af5d6dfa2310b1 Mon Sep 17 00:00:00 2001
From: nuno1212s <nuno.neto.g@gmail.com>
Date: Mon, 19 Jun 2023 21:26:47 +0100
Subject: [PATCH 04/80] Fixed compilation issues with ordering protocol.

---
 febft-pbft-consensus/src/bft/config/mod.rs    |   6 +-
 .../src/bft/consensus/accessory/mod.rs        |  46 ++++----
 .../bft/consensus/accessory/replica/mod.rs    |  56 +++++-----
 .../src/bft/consensus/decision/mod.rs         |  33 +++---
 febft-pbft-consensus/src/bft/consensus/mod.rs |  48 ++++----
 febft-pbft-consensus/src/bft/message/mod.rs   |   2 +-
 .../src/bft/message/serialize/capnp/mod.rs    |  18 +--
 .../src/bft/message/serialize/mod.rs          |  14 +--
 .../src/bft/message/serialize/serde/mod.rs    |   6 +-
 febft-pbft-consensus/src/bft/mod.rs           |  78 ++++++++-----
 .../src/bft/msg_log/decided_log/mod.rs        |   8 +-
 .../src/bft/msg_log/decisions/mod.rs          |   8 +-
 febft-pbft-consensus/src/bft/msg_log/mod.rs   |   8 +-
 .../src/bft/proposer/follower_proposer/mod.rs |  23 ++--
 febft-pbft-consensus/src/bft/proposer/mod.rs  |  36 +++---
 .../src/bft/sync/follower_sync/mod.rs         |   8 +-
 febft-pbft-consensus/src/bft/sync/mod.rs      | 104 ++++++++++--------
 .../src/bft/sync/replica_sync/mod.rs          |  32 +++---
 febft-state-transfer/src/lib.rs               |  28 ++---
 .../src/message/serialize/capnp/mod.rs        |   6 +-
 .../src/message/serialize/mod.rs              |   2 +-
 21 files changed, 314 insertions(+), 256 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/config/mod.rs b/febft-pbft-consensus/src/bft/config/mod.rs
index f59b4a5c..9c4a8f01 100644
--- a/febft-pbft-consensus/src/bft/config/mod.rs
+++ b/febft-pbft-consensus/src/bft/config/mod.rs
@@ -5,7 +5,7 @@ use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
 use atlas_communication::Node;
 use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::followers::FollowerHandle;
 use atlas_core::serialize::{OrderingProtocolMessage, StateTransferMessage, ServiceMsg};
 use atlas_core::state_transfer::Checkpoint;
@@ -14,7 +14,7 @@ use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::observer::ObserverHandle;
 use crate::bft::sync::view::ViewInfo;
 
-pub struct PBFTConfig<D: SharedData, ST> {
+pub struct PBFTConfig<D: ApplicationData, ST> {
     pub node_id: NodeId,
     // pub observer_handle: ObserverHandle,
     pub follower_handle: Option<FollowerHandle<PBFTConsensus<D>>>,
@@ -25,7 +25,7 @@ pub struct PBFTConfig<D: SharedData, ST> {
     pub _phantom_data: PhantomData<ST>,
 }
 
-impl<D: SharedData + 'static,
+impl<D: ApplicationData + 'static,
     ST: StateTransferMessage + 'static> PBFTConfig<D, ST> {
     pub fn new(node_id: NodeId,
                follower_handle: Option<FollowerHandle<PBFTConsensus<D>>>,
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
index 94e2304a..a0412e05 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
@@ -1,8 +1,8 @@
 use atlas_common::crypto::hash::Digest;
 use atlas_common::ordering::SeqNo;
 use atlas_communication::Node;
-use atlas_execution::serialize::SharedData;
-use atlas_core::serialize::StateTransferMessage;
+use atlas_execution::serialize::ApplicationData;
+use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
 use crate::bft::message::ConsensusMessage;
 use crate::bft::msg_log::deciding_log::DecidingLog;
@@ -12,61 +12,63 @@ use crate::bft::sync::view::ViewInfo;
 
 pub mod replica;
 
-pub enum ConsensusDecisionAccessory<D: SharedData + 'static, ST: StateTransferMessage + 'static> {
+pub enum ConsensusDecisionAccessory<D, ST, LP>
+    where D: ApplicationData + 'static, ST: StateTransferMessage + 'static, LP: LogTransferMessage + 'static {
     Follower,
-    Replica(ReplicaAccessory<D, ST>),
+    Replica(ReplicaAccessory<D, ST, LP>),
 }
 
-pub trait AccessoryConsensus<D, ST> where D: SharedData + 'static,
-                                          ST: StateTransferMessage + 'static {
-
+pub trait AccessoryConsensus<D, ST, LP> where D: ApplicationData + 'static,
+                                              ST: StateTransferMessage + 'static,
+                                              LP: LogTransferMessage + 'static {
     /// Handle the reception of a pre-prepare message without having completed the pre prepare phase
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: Node<PBFT<D, ST>>;
+                                      node: &NT) where NT: Node<PBFT<D, ST, LP>>;
 
     /// Handle the prepare phase having been completed
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
-                                              node: &NT) where NT: Node<PBFT<D, ST>>;
+                                              node: &NT) where NT: Node<PBFT<D, ST, LP>>;
 
     /// Handle a prepare message processed during the preparing phase without having
     /// reached a quorum
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: Node<PBFT<D, ST>>;
+                                      node: &NT) where NT: Node<PBFT<D, ST, LP>>;
 
     /// Handle a prepare message processed during the prepare phase when a quorum
     /// has been achieved
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
-                                   node: &NT) where NT: Node<PBFT<D, ST>>;
+                                   node: &NT) where NT: Node<PBFT<D, ST, LP>>;
 
     /// Handle a commit message processed during the preparing phase without having
     /// reached a quorum
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
-                                       node: &NT) where NT: Node<PBFT<D, ST>>;
+                                       node: &NT) where NT: Node<PBFT<D, ST, LP>>;
 
     /// Handle a commit message processed during the prepare phase when a quorum has been reached
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
-                                    node: &NT) where NT: Node<PBFT<D, ST>>;
+                                    node: &NT) where NT: Node<PBFT<D, ST, LP>>;
 }
 
-impl<D, ST> AccessoryConsensus<D, ST> for ConsensusDecisionAccessory<D, ST>
-    where D: SharedData + 'static, ST: StateTransferMessage + 'static {
-
+impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D, ST, LP>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static {
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: Node<PBFT<D, ST>> {
+                                      node: &NT) where NT: Node<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -78,7 +80,7 @@ impl<D, ST> AccessoryConsensus<D, ST> for ConsensusDecisionAccessory<D, ST>
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
-                                              node: &NT) where NT: Node<PBFT<D, ST>> {
+                                              node: &NT) where NT: Node<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -90,7 +92,7 @@ impl<D, ST> AccessoryConsensus<D, ST> for ConsensusDecisionAccessory<D, ST>
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: Node<PBFT<D, ST>> {
+                                      node: &NT) where NT: Node<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -102,7 +104,7 @@ impl<D, ST> AccessoryConsensus<D, ST> for ConsensusDecisionAccessory<D, ST>
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
-                                   node: &NT) where NT: Node<PBFT<D, ST>> {
+                                   node: &NT) where NT: Node<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -114,7 +116,7 @@ impl<D, ST> AccessoryConsensus<D, ST> for ConsensusDecisionAccessory<D, ST>
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
-                                       node: &NT) where NT: Node<PBFT<D, ST>> {
+                                       node: &NT) where NT: Node<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -126,7 +128,7 @@ impl<D, ST> AccessoryConsensus<D, ST> for ConsensusDecisionAccessory<D, ST>
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
-                                    node: &NT) where NT: Node<PBFT<D, ST>> {
+                                    node: &NT) where NT: Node<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index d41947e2..fbbea87d 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -10,9 +10,9 @@ use atlas_common::threadpool;
 use atlas_communication::message::{NetworkMessageKind, SerializedMessage, StoredMessage, StoredSerializedNetworkMessage, WireMessage};
 use atlas_communication::{Node, NodePK, serialize};
 use atlas_communication::serialize::Buf;
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::SystemMessage;
-use atlas_core::serialize::StateTransferMessage;
+use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::msg_log::deciding_log::DecidingLog;
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
@@ -20,24 +20,27 @@ use crate::bft::PBFT;
 use crate::bft::sync::view::ViewInfo;
 use crate::bft::consensus::accessory::AccessoryConsensus;
 
-pub struct ReplicaAccessory<D: SharedData + 'static, ST: StateTransferMessage + 'static> {
-    speculative_commits: Arc<Mutex<BTreeMap<NodeId, StoredSerializedNetworkMessage<PBFT<D, ST>>>>>,
+pub struct ReplicaAccessory<D, ST, LP>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static {
+    speculative_commits: Arc<Mutex<BTreeMap<NodeId, StoredSerializedNetworkMessage<PBFT<D, ST, LP>>>>>,
 }
 
-impl<D, ST> AccessoryConsensus<D, ST> for ReplicaAccessory<D, ST>
-    where D: SharedData + 'static,
-          ST: StateTransferMessage + 'static {
-
+impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static {
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: Node<PBFT<D, ST>> {}
+                                      node: &NT) where NT: Node<PBFT<D, ST, LP>> {}
 
     fn handle_pre_prepare_phase_completed<NT>(&mut self,
                                               deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               _msg: StoredConsensusMessage<D::Request>,
-                                              node: &NT) where NT: Node<PBFT<D, ST>> {
+                                              node: &NT) where NT: Node<PBFT<D, ST, LP>> {
         let my_id = node.id();
         let view_seq = view.sequence_number();
         let current_digest = deciding_log.current_digest().unwrap();
@@ -63,7 +66,7 @@ impl<D, ST> AccessoryConsensus<D, ST> for ReplicaAccessory<D, ST>
             let mut buf = Vec::new();
 
             let digest = serialize::serialize_digest::<Vec<u8>,
-                PBFT<D, ST>>(&message, &mut buf).unwrap();
+                PBFT<D, ST, LP>>(&message, &mut buf).unwrap();
 
             let buf = Buf::from(buf);
 
@@ -117,19 +120,18 @@ impl<D, ST> AccessoryConsensus<D, ST> for ReplicaAccessory<D, ST>
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST>> {}
+                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST, LP>> {}
 
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
-                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST>> {
-
+                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST, LP>> {
         let node_id = node.id();
 
         let seq = deciding_log.sequence_number();
         let current_digest = deciding_log.current_digest().unwrap();
         let speculative_commits = self.take_speculative_commits();
 
-        if valid_spec_commits::<D, ST>(&speculative_commits, node_id, seq, view) {
+        if valid_spec_commits::<D, ST, LP>(&speculative_commits, node_id, seq, view) {
             for (_, msg) in speculative_commits.iter() {
                 debug!("{:?} // Broadcasting speculative commit message {:?} (total of {} messages) to {} targets",
                      node_id, msg.message().original(), speculative_commits.len(), view.params().n());
@@ -160,41 +162,41 @@ impl<D, ST> AccessoryConsensus<D, ST> for ReplicaAccessory<D, ST>
 
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
-                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST>> {}
+                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST, LP>> {}
 
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
-                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST>> {}
-
+                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST, LP>> {}
 }
 
-impl<D, ST> ReplicaAccessory<D, ST>
-    where D: SharedData + 'static,
-          ST: StateTransferMessage + 'static {
+impl<D, ST, LP> ReplicaAccessory<D, ST, LP>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static {
     pub fn new() -> Self {
         Self {
             speculative_commits: Arc::new(Mutex::new(BTreeMap::new())),
         }
     }
 
-    fn take_speculative_commits(&self) -> BTreeMap<NodeId, StoredSerializedNetworkMessage<PBFT<D, ST>>> {
+    fn take_speculative_commits(&self) -> BTreeMap<NodeId, StoredSerializedNetworkMessage<PBFT<D, ST, LP>>> {
         let mut map = self.speculative_commits.lock().unwrap();
         std::mem::replace(&mut *map, BTreeMap::new())
     }
-
 }
 
 
 #[inline]
-fn valid_spec_commits<D, ST>(
-    speculative_commits: &BTreeMap<NodeId, StoredSerializedNetworkMessage<PBFT<D, ST>>>,
+fn valid_spec_commits<D, ST, LP>(
+    speculative_commits: &BTreeMap<NodeId, StoredSerializedNetworkMessage<PBFT<D, ST, LP>>>,
     node_id: NodeId,
     seq_no: SeqNo,
     view: &ViewInfo,
 ) -> bool
     where
-        D: SharedData + 'static,
-        ST: StateTransferMessage
+        D: ApplicationData + 'static,
+        ST: StateTransferMessage + 'static,
+        LP: LogTransferMessage + 'static
 {
     let len = speculative_commits.len();
 
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index beff3a20..7da99d39 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -13,8 +13,8 @@ use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::Node;
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, WriteMode};
-use atlas_execution::serialize::SharedData;
-use atlas_core::serialize::StateTransferMessage;
+use atlas_execution::serialize::ApplicationData;
+use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
 use atlas_metrics::metrics::{metric_duration};
 use crate::bft::consensus::accessory::{AccessoryConsensus, ConsensusDecisionAccessory};
@@ -102,7 +102,9 @@ pub struct MessageQueue<O> {
 }
 
 /// The information needed to make a decision on a batch of requests.
-pub struct ConsensusDecision<D: SharedData + 'static, ST: StateTransferMessage + 'static, PL> {
+pub struct ConsensusDecision<D, ST, LP, PL> where D: ApplicationData + 'static,
+                                                  ST: StateTransferMessage + 'static,
+                                                  LP: LogTransferMessage + 'static {
     node_id: NodeId,
     /// The sequence number of this consensus decision
     seq: SeqNo,
@@ -115,7 +117,7 @@ pub struct ConsensusDecision<D: SharedData + 'static, ST: StateTransferMessage +
     /// Persistent log reference
     persistent_log: PL,
     /// Accessory to the base consensus state machine
-    accessory: ConsensusDecisionAccessory<D, ST>,
+    accessory: ConsensusDecisionAccessory<D, ST, LP>,
     // Metrics about the consensus instance
     consensus_metrics: ConsensusMetrics,
     //TODO: Store things directly into the persistent log as well as delete them when
@@ -171,7 +173,10 @@ impl<O> MessageQueue<O> {
     }
 }
 
-impl<D: SharedData + 'static, ST: StateTransferMessage + 'static, PL> ConsensusDecision<D, ST, PL> {
+impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static {
     pub fn init_decision(node_id: NodeId, seq_no: SeqNo, view: &ViewInfo, persistent_log: PL) -> Self {
         Self {
             node_id,
@@ -257,13 +262,13 @@ impl<D: SharedData + 'static, ST: StateTransferMessage + 'static, PL> ConsensusD
 
     /// Process a message relating to this consensus instance
     pub fn process_message<NT>(&mut self,
-                                   header: Header,
-                                   message: ConsensusMessage<D::Request>,
-                                   synchronizer: &Synchronizer<D>,
-                                   timeouts: &Timeouts,
-                                   log: &mut Log<D, PL>,
-                                   node: &NT) -> Result<DecisionStatus>
-        where NT: Node<PBFT<D, ST>>,
+                               header: Header,
+                               message: ConsensusMessage<D::Request>,
+                               synchronizer: &Synchronizer<D>,
+                               timeouts: &Timeouts,
+                               log: &mut Log<D, PL>,
+                               node: &NT) -> Result<DecisionStatus>
+        where NT: Node<PBFT<D, ST, LP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let view = synchronizer.view();
 
@@ -591,7 +596,7 @@ impl<D: SharedData + 'static, ST: StateTransferMessage + 'static, PL> ConsensusD
     }
 }
 
-impl<D: SharedData + 'static, ST: StateTransferMessage + 'static, PL> Orderable for ConsensusDecision<D, ST, PL> {
+impl<D: ApplicationData + 'static, ST: StateTransferMessage + 'static, LP: LogTransferMessage + 'static, PL> Orderable for ConsensusDecision<D, ST, LP, PL> {
     fn sequence_number(&self) -> SeqNo {
         self.seq
     }
@@ -605,7 +610,7 @@ fn request_batch_received<D>(
     log: &DecidingLog<D::Request>,
 ) -> Vec<ClientRqInfo>
     where
-        D: SharedData + 'static
+        D: ApplicationData + 'static
 {
     let start = Instant::now();
 
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 1c7d5e7b..479874dc 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -18,11 +18,11 @@ use atlas_common::ordering::{InvalidSeqNo, Orderable, SeqNo, tbo_advance_message
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::Node;
 use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage, SystemMessage};
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_core::serialize::StateTransferMessage;
+use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
 use atlas_metrics::metrics::metric_increment;
 use crate::bft;
@@ -186,7 +186,11 @@ pub struct Signals {
 
 /// The consensus handler. Responsible for multiplexing consensus instances and keeping track
 /// of missing messages
-pub struct Consensus<D: SharedData + 'static, ST: StateTransferMessage + 'static, PL: Clone> {
+pub struct Consensus<D, ST, LP, PL>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static,
+          PL: Clone {
     node_id: NodeId,
     /// The handle to the executor of the function
     executor_handle: ExecutorHandle<D>,
@@ -201,7 +205,7 @@ pub struct Consensus<D: SharedData + 'static, ST: StateTransferMessage + 'static
     /// The consensus instances that are currently being processed
     /// A given consensus instance n will only be finished when all consensus instances
     /// j, where j < n have already been processed, in order to maintain total ordering
-    decisions: VecDeque<ConsensusDecision<D, ST, PL>>,
+    decisions: VecDeque<ConsensusDecision<D, ST, LP, PL>>,
     /// The queue for messages that sit outside the range seq_no + watermark
     /// These messages cannot currently be processed since they sit outside the allowed
     /// zone but they will be processed once the seq no moves forward enough to include them
@@ -220,9 +224,10 @@ pub struct Consensus<D: SharedData + 'static, ST: StateTransferMessage + 'static
     persistent_log: PL,
 }
 
-impl<D, ST, PL> Consensus<D, ST, PL> where D: SharedData + 'static,
-                                           ST: StateTransferMessage + 'static,
-                                           PL: Clone {
+impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
+                                                   ST: StateTransferMessage + 'static,
+                                                   LP: LogTransferMessage + 'static,
+                                                   PL: Clone {
     pub fn new_replica(node_id: NodeId, view: &ViewInfo, executor_handle: ExecutorHandle<D>, seq_no: SeqNo,
                        watermark: u32, consensus_guard: Arc<ProposerConsensusGuard>, timeouts: Timeouts,
                        persistent_log: PL) -> Self {
@@ -361,13 +366,13 @@ impl<D, ST, PL> Consensus<D, ST, PL> where D: SharedData + 'static,
     }
 
     pub fn process_message<NT>(&mut self,
-                               header: Header,
-                               message: ConsensusMessage<D::Request>,
-                               synchronizer: &Synchronizer<D>,
-                               timeouts: &Timeouts,
-                               log: &mut Log<D, PL>,
-                               node: &NT) -> Result<ConsensusStatus>
-        where NT: Node<PBFT<D, ST>>,
+                                   header: Header,
+                                   message: ConsensusMessage<D::Request>,
+                                   synchronizer: &Synchronizer<D>,
+                                   timeouts: &Timeouts,
+                                   log: &mut Log<D, PL>,
+                                   node: &NT) -> Result<ConsensusStatus>
+        where NT: Node<PBFT<D, ST, LP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let message_seq = message.sequence_number();
 
@@ -480,7 +485,7 @@ impl<D, ST, PL> Consensus<D, ST, PL> where D: SharedData + 'static,
     /// Advance to the next instance of the consensus
     /// This will also create the necessary new decision to keep the pending decisions
     /// equal to the water mark
-    pub fn next_instance(&mut self, view: &ViewInfo) -> ConsensusDecision<D, ST, PL> {
+    pub fn next_instance(&mut self, view: &ViewInfo) -> ConsensusDecision<D, ST, LP, PL> {
         let decision = self.decisions.pop_front().unwrap();
 
         self.seq_no = self.seq_no.next();
@@ -715,7 +720,7 @@ impl<D, ST, PL> Consensus<D, ST, PL> where D: SharedData + 'static,
         &self,
         requests: Vec<StoredRequestMessage<D::Request>>,
         synchronizer: &K,
-    ) -> SysMsg<D, ST>
+    ) -> SysMsg<D, ST, LP>
         where
             K: AbstractSynchronizer<D>,
     {
@@ -802,7 +807,7 @@ impl<D, ST, PL> Consensus<D, ST, PL> where D: SharedData + 'static,
         timeouts: &Timeouts,
         log: &mut Log<D, PL>,
         node: &NT,
-    ) where NT: Node<PBFT<D, ST>>,
+    ) where NT: Node<PBFT<D, ST, LP>>,
             PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let view = synchronizer.view();
         //Prepare the algorithm as we are already entering this phase
@@ -845,7 +850,7 @@ impl<D, ST, PL> Consensus<D, ST, PL> where D: SharedData + 'static,
     }
 
     /// Enqueue a decision onto our overlapping decision log
-    fn enqueue_decision(&mut self, decision: ConsensusDecision<D, ST, PL>) {
+    fn enqueue_decision(&mut self, decision: ConsensusDecision<D, ST, LP, PL>) {
         self.signalled.push_signalled(decision.sequence_number());
 
         self.decisions.push_back(decision);
@@ -867,8 +872,11 @@ impl<D, ST, PL> Consensus<D, ST, PL> where D: SharedData + 'static,
     }
 }
 
-impl<D, ST, PL> Orderable for Consensus<D, ST, PL> where D: SharedData + 'static, ST: StateTransferMessage + 'static,
-                                                         PL: Clone  {
+impl<D, ST, LP, PL> Orderable for Consensus<D, ST, LP, PL>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static,
+          PL: Clone {
     fn sequence_number(&self) -> SeqNo {
         self.seq_no
     }
diff --git a/febft-pbft-consensus/src/bft/message/mod.rs b/febft-pbft-consensus/src/bft/message/mod.rs
index e1d040fb..b31ab93e 100644
--- a/febft-pbft-consensus/src/bft/message/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/mod.rs
@@ -20,7 +20,7 @@ use atlas_common::crypto::hash::{Context, Digest};
 use atlas_common::crypto::signature::{KeyPair, PublicKey, Signature};
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, NetworkMessage, NetworkMessageKind, PingMessage, StoredMessage};
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{RequestMessage, StoredRequestMessage};
 
 use crate::bft::sync::LeaderCollects;
diff --git a/febft-pbft-consensus/src/bft/message/serialize/capnp/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/capnp/mod.rs
index 97891683..b0b58719 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/capnp/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/capnp/mod.rs
@@ -15,7 +15,7 @@ use crate::bft::message::{ConsensusMessage, ConsensusMessageKind,  ObserveEventK
 use crate::bft::msg_log::persistent::ProofInfo;
 use crate::bft::sync::view::ViewInfo;
 
-use super::{Buf, SharedData};
+use super::{Buf, ApplicationData};
 
 /// This module is meant to handle the serialization of the SMR messages, to allow for the users of this library to only
 /// Have to serialize (and declare) their request, reply and state, instead of also having to do so for all message
@@ -24,7 +24,7 @@ use super::{Buf, SharedData};
 const DEFAULT_SERIALIZE_BUFFER_SIZE: usize = 1024;
 
 pub fn serialize_message<D>(mut pbft_message: consensus_messages_capnp::protocol_message::Builder,
-                            m: &PBFTMessage<D::Request>) -> Result<()> where D: SharedData {
+                            m: &PBFTMessage<D::Request>) -> Result<()> where D: ApplicationData {
     match m {
         PBFTMessage::Consensus(consensus_msg) => {
             let mut consensus_builder = pbft_message.init_consensus_message();
@@ -48,7 +48,7 @@ pub fn serialize_message<D>(mut pbft_message: consensus_messages_capnp::protocol
 }
 
 pub fn deserialize_message<D>(pbft_reader: consensus_messages_capnp::protocol_message::Reader) -> Result<PBFTMessage<D::Request>> where
-    D: SharedData {
+    D: ApplicationData {
     let which = pbft_reader.which().wrapped(ErrorKind::CommunicationSerialize)?;
 
     match which {
@@ -77,7 +77,7 @@ fn deserialize_consensus_message<D>(
     consensus_msg: consensus_messages_capnp::consensus::Reader,
 ) -> Result<ConsensusMessage<D::Request>>
     where
-        D: SharedData,
+        D: ApplicationData,
 {
     let seq_no: SeqNo = consensus_msg.get_seq_no().into();
     let view: SeqNo = consensus_msg.get_view().into();
@@ -138,7 +138,7 @@ fn deserialize_consensus_message<D>(
 pub fn deserialize_consensus<R, D>(r: R) -> Result<ConsensusMessage<D::Request>>
     where
         R: Read,
-        D: SharedData,
+        D: ApplicationData,
 {
     let reader = capnp::serialize::read_message(r, Default::default()).wrapped_msg(
         ErrorKind::CommunicationSerialize,
@@ -158,7 +158,7 @@ fn serialize_consensus_message<S>(
     m: &ConsensusMessage<S::Request>,
 ) -> Result<()>
     where
-        S: SharedData,
+        S: ApplicationData,
 {
     consensus.set_seq_no(m.sequence_number().into());
     consensus.set_view(m.view().into());
@@ -206,7 +206,7 @@ fn serialize_consensus_message<S>(
 pub fn serialize_consensus<W, S>(w: &mut W, message: &ConsensusMessage<S::Request>) -> Result<()>
     where
         W: Write,
-        S: SharedData,
+        S: ApplicationData,
 {
     let mut root = capnp::message::Builder::new(capnp::message::HeapAllocator::new());
 
@@ -221,13 +221,13 @@ pub fn serialize_consensus<W, S>(w: &mut W, message: &ConsensusMessage<S::Reques
 }
 
 fn serialize_view_change<D>(mut view_change: consensus_messages_capnp::view_change::Builder,
-                            msg: &ViewChangeMessage<D::Request>) -> Result<()> where D: SharedData {
+                            msg: &ViewChangeMessage<D::Request>) -> Result<()> where D: ApplicationData {
     Ok(())
 }
 
 fn deserialize_view_change<D>(view_change: consensus_messages_capnp::view_change::Reader)
                               -> Result<ViewChangeMessage<D::Request>>
-    where D: SharedData {
+    where D: ApplicationData {
     Err(Error::simple(ErrorKind::CommunicationSerialize))
 }
 
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index b361cd3a..24937173 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -21,10 +21,8 @@ use atlas_communication::Node;
 use atlas_communication::serialize::Serializable;
 use atlas_core::ordering_protocol::{ProtocolMessage, SerProof, SerProofMetadata};
 use atlas_core::persistent_log::PersistableOrderProtocol;
-use atlas_execution::app::Service;
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::serialize::{OrderingProtocolMessage, StatefulOrderProtocolMessage};
-use atlas_core::state_transfer::DecLog;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::{PBFT};
 use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata};
@@ -42,7 +40,7 @@ pub type Buf = Bytes;
 pub fn serialize_consensus<W, D>(w: &mut W, message: &ConsensusMessage<D::Request>) -> Result<()>
     where
         W: Write + AsRef<[u8]> + AsMut<[u8]>,
-        D: SharedData,
+        D: ApplicationData,
 {
     #[cfg(feature = "serialize_capnp")]
     capnp::serialize_consensus::<W, D>(w, message)?;
@@ -56,7 +54,7 @@ pub fn serialize_consensus<W, D>(w: &mut W, message: &ConsensusMessage<D::Reques
 pub fn deserialize_consensus<R, D>(r: R) -> Result<ConsensusMessage<D::Request>>
     where
         R: Read + AsRef<[u8]>,
-        D: SharedData,
+        D: ApplicationData,
 {
     #[cfg(feature = "serialize_capnp")]
         let result = capnp::deserialize_consensus::<R, D>(r)?;
@@ -68,9 +66,9 @@ pub fn deserialize_consensus<R, D>(r: R) -> Result<ConsensusMessage<D::Request>>
 }
 
 /// The serializable type, to be used to appease the compiler and it's requirements
-pub struct PBFTConsensus<D: SharedData>(PhantomData<D>);
+pub struct PBFTConsensus<D: ApplicationData>(PhantomData<D>);
 
-impl<D> OrderingProtocolMessage for PBFTConsensus<D> where D: SharedData {
+impl<D> OrderingProtocolMessage for PBFTConsensus<D> where D: ApplicationData {
     type ViewInfo = ViewInfo;
     type ProtocolMessage = PBFTMessage<D::Request>;
     type Proof = Proof<D::Request>;
@@ -107,7 +105,7 @@ impl<D> OrderingProtocolMessage for PBFTConsensus<D> where D: SharedData {
     }
 }
 
-impl<D> StatefulOrderProtocolMessage for PBFTConsensus<D> where D: SharedData {
+impl<D> StatefulOrderProtocolMessage for PBFTConsensus<D> where D: ApplicationData {
     type DecLog = DecisionLog<D::Request>;
 
     #[cfg(feature = "serialize_capnp")]
diff --git a/febft-pbft-consensus/src/bft/message/serialize/serde/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/serde/mod.rs
index 1849489c..38bd5111 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/serde/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/serde/mod.rs
@@ -1,6 +1,6 @@
 use std::io::{Read, Write};
 use atlas_common::error::*;
-use crate::bft::message::serialize::SharedData;
+use crate::bft::message::serialize::ApplicationData;
 use crate::bft::message::{ConsensusMessage};
 
 pub fn serialize_consensus<W, D>(
@@ -8,7 +8,7 @@ pub fn serialize_consensus<W, D>(
     w: &mut W,
 ) -> Result<()> where
     W: Write + AsMut<[u8]>,
-    D: SharedData {
+    D: ApplicationData {
 
     bincode::serde::encode_into_std_write(m,  w, bincode::config::standard())
         .wrapped_msg(ErrorKind::CommunicationSerialize, format!("Failed to serialize message {} bytes len", w.as_mut().len()).as_str())?;
@@ -18,7 +18,7 @@ pub fn serialize_consensus<W, D>(
 
 pub fn deserialize_consensus<R, D>(
     r: R
-) -> Result<ConsensusMessage<D::Request>> where D: SharedData, R: Read + AsRef<[u8]> {
+) -> Result<ConsensusMessage<D::Request>> where D: ApplicationData, R: Read + AsRef<[u8]> {
     let msg =  bincode::serde::decode_borrowed_from_slice(r.as_ref(), bincode::config::standard())
         .wrapped_msg(ErrorKind::CommunicationSerialize, "Failed to deserialize message")?;
 
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 4c86185c..9f776096 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -24,14 +24,15 @@ use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::Node;
 use atlas_communication::serialize::Serializable;
 use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::messages::Protocol;
 use atlas_core::ordering_protocol::{OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, ProtocolConsensusDecision, ProtocolMessage, SerProof, SerProofMetadata, View};
 use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol, StatefulOrderingProtocolLog};
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
-use atlas_core::serialize::{NetworkView, ServiceMsg, StateTransferMessage};
-use atlas_core::state_transfer::{Checkpoint, DecLog, StatefulOrderProtocol};
+use atlas_core::serialize::{LogTransferMessage, NetworkView, ServiceMsg, StateTransferMessage};
+use atlas_core::state_transfer::{Checkpoint};
+use atlas_core::state_transfer::log_transfer::{DecLog, StatefulOrderProtocol};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_metrics::metrics::metric_duration;
 use crate::bft::config::PBFTConfig;
@@ -55,9 +56,9 @@ pub mod observer;
 pub mod metric;
 
 // The types responsible for this protocol
-pub type PBFT<D, ST> = ServiceMsg<D, PBFTConsensus<D>, ST>;
+pub type PBFT<D, ST, LP> = ServiceMsg<D, PBFTConsensus<D>, ST, LP>;
 // The message type for this consensus protocol
-pub type SysMsg<D, ST> = <PBFT<D, ST> as Serializable>::Message;
+pub type SysMsg<D, ST, LP> = <PBFT<D, ST, LP> as Serializable>::Message;
 
 #[derive(Clone, PartialEq, Eq, Debug)]
 /// Which phase of the consensus algorithm are we currently executing
@@ -78,17 +79,18 @@ pub enum SyncPhaseRes<O> {
 }
 
 /// a PBFT based ordering protocol
-pub struct PBFTOrderProtocol<D, ST, NT, PL>
+pub struct PBFTOrderProtocol<D, ST, LP, NT, PL>
     where
-        D: SharedData + 'static,
+        D: ApplicationData + 'static,
         ST: StateTransferMessage + 'static,
-        NT: Node<PBFT<D, ST>> + 'static,
+        LP: LogTransferMessage + 'static,
+        NT: Node<PBFT<D, ST, LP>> + 'static,
         PL: Clone {
     // What phase of the consensus algorithm are we currently executing
     phase: ConsensusPhase,
 
     /// The consensus state machine
-    consensus: Consensus<D, ST, PL>,
+    consensus: Consensus<D, ST, LP, PL>,
     /// The synchronizer state machine
     synchronizer: Arc<Synchronizer<D>>,
     /// The request pre processor
@@ -113,19 +115,21 @@ pub struct PBFTOrderProtocol<D, ST, NT, PL>
     executor: ExecutorHandle<D>,
 }
 
-impl<D, ST, NT, PL> Orderable for PBFTOrderProtocol<D, ST, NT, PL> where D: 'static + SharedData,
-                                                                         NT: 'static + Node<PBFT<D, ST>>,
-                                                                         ST: 'static + StateTransferMessage,
-                                                                         PL: Clone {
+impl<D, ST, LP, NT, PL> Orderable for PBFTOrderProtocol<D, ST, LP, NT, PL> where D: 'static + ApplicationData,
+                                                                                 NT: 'static + Node<PBFT<D, ST, LP>>,
+                                                                                 ST: 'static + StateTransferMessage,
+                                                                                 LP: 'static + LogTransferMessage,
+                                                                                 PL: Clone {
     fn sequence_number(&self) -> SeqNo {
         self.consensus.sequence_number()
     }
 }
 
-impl<D, ST, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, NT, PL>
-    where D: SharedData + 'static,
+impl<D, ST, LP, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, LP, NT, PL>
+    where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          NT: Node<PBFT<D, ST>> + 'static,
+          LP: LogTransferMessage + 'static,
+          NT: Node<PBFT<D, ST, LP>> + 'static,
           PL: Clone {
     type Serialization = PBFTConsensus<D>;
     type Config = PBFTConfig<D, ST>;
@@ -260,10 +264,11 @@ impl<D, ST, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, NT,
     }
 }
 
-impl<D, ST, NT, PL> PBFTOrderProtocol<D, ST, NT, PL> where D: SharedData + 'static,
-                                                           ST: StateTransferMessage + 'static,
-                                                           NT: Node<PBFT<D, ST>> + 'static,
-                                                           PL: Clone {
+impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: ApplicationData + 'static,
+                                                                   ST: StateTransferMessage + 'static,
+                                                                   LP: LogTransferMessage + 'static,
+                                                                   NT: Node<PBFT<D, ST, LP>> + 'static,
+                                                                   PL: Clone {
     fn initialize_protocol(config: PBFTConfig<D, ST>, args: OrderingProtocolArgs<D, NT, PL>,
                            initial_state: Option<DecisionLog<D::Request>>) -> Result<Self> {
         let PBFTConfig {
@@ -280,7 +285,7 @@ impl<D, ST, NT, PL> PBFTOrderProtocol<D, ST, NT, PL> where D: SharedData + 'stat
 
         let consensus_guard = ProposerConsensusGuard::new(view.clone(), watermark);
 
-        let consensus = Consensus::<D, ST, PL>::new_replica(node_id, &sync.view(), executor.clone(),
+        let consensus = Consensus::<D, ST, LP, PL>::new_replica(node_id, &sync.view(), executor.clone(),
                                                             SeqNo::ZERO, watermark, consensus_guard.clone(),
                                                             timeouts.clone(), persistent_log.clone());
 
@@ -599,10 +604,11 @@ impl<D, ST, NT, PL> PBFTOrderProtocol<D, ST, NT, PL> where D: SharedData + 'stat
     }
 }
 
-impl<D, ST, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, NT, PL>
-    where D: SharedData + 'static,
+impl<D, ST, LP, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, LP, NT, PL>
+    where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          NT: Node<PBFT<D, ST>> + 'static,
+          LP: LogTransferMessage + 'static,
+          NT: Node<PBFT<D, ST, LP>> + 'static,
           PL: Clone {
     type StateSerialization = PBFTConsensus<D>;
 
@@ -648,15 +654,25 @@ impl<D, ST, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST
         self.message_log.snapshot(self.synchronizer.view())
     }
 
+    fn current_log(&self) -> Result<&DecLog<Self::StateSerialization>>
+        where PL: StatefulOrderingProtocolLog<Self::Serialization, Self::StateSerialization> {
+        Ok(self.message_log.decision_log())
+    }
+
     fn checkpointed(&mut self, seq_no: SeqNo) -> Result<()> {
         self.message_log.finalize_checkpoint(seq_no)
     }
+
+    fn get_proof(&self, seq: SeqNo) -> Result<Option<SerProof<Self::Serialization>>> {
+        todo!()
+    }
 }
 
-impl<D, ST, NT, PL> PBFTOrderProtocol<D, ST, NT, PL>
-    where D: SharedData + 'static,
+impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL>
+    where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          NT: Node<PBFT<D, ST>> + 'static,
+          LP: LogTransferMessage + 'static,
+          NT: Node<PBFT<D, ST, LP>> + 'static,
           PL: Clone {
     pub(crate) fn switch_phase(&mut self, new_phase: ConsensusPhase) {
         info!("{:?} // Switching from phase {:?} to phase {:?}", self.node.id(), self.phase, new_phase);
@@ -716,9 +732,11 @@ const CF_PRE_PREPARES: &str = "PRE_PREPARES";
 const CF_PREPARES: &str = "PREPARES";
 const CF_COMMIT: &str = "COMMITS";
 
-impl<D, ST, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus<D>> for PBFTOrderProtocol<D, ST, NT, PL>
-    where D: SharedData, ST: StateTransferMessage, NT: Node<PBFT<D, ST>>, PL: Clone {
-
+impl<D, ST, LP, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus<D>> for PBFTOrderProtocol<D, ST, LP, NT, PL>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static,
+          NT: Node<PBFT<D, ST, LP>>, PL: Clone {
     fn message_types() -> Vec<&'static str> {
         vec![
             CF_PRE_PREPARES,
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index ae9f7d82..0a8ae598 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -12,8 +12,8 @@ use atlas_communication::Node;
 use atlas_core::ordering_protocol::{DecisionInformation, ExecutionResult, ProtocolConsensusDecision};
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, WriteMode};
 use atlas_core::serialize::StateTransferMessage;
-use atlas_execution::app::{Request, Service, State, UpdateBatch};
-use atlas_execution::serialize::SharedData;
+use atlas_execution::app::{Request, UpdateBatch};
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::state_transfer::Checkpoint;
 
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
@@ -26,7 +26,7 @@ use crate::bft::sync::view::ViewInfo;
 
 /// The log of decisions that have already been processed by the consensus
 /// algorithm
-pub struct Log<D, PL> where D: SharedData + 'static {
+pub struct Log<D, PL> where D: ApplicationData + 'static {
     // The log for all of the already decided consensus instances
     dec_log: DecisionLog<D::Request>,
 
@@ -34,7 +34,7 @@ pub struct Log<D, PL> where D: SharedData + 'static {
     persistent_log: PL,
 }
 
-impl<D, PL> Log<D, PL> where D: SharedData + 'static {
+impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     pub(crate) fn init_decided_log(node_id: NodeId, persistent_log: PL,
                                    dec_log: Option<DecisionLog<D::Request>>) -> Self {
         Self {
diff --git a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
index d55247ae..ecdb243c 100644
--- a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
@@ -11,7 +11,7 @@ use atlas_common::globals::ReadOnly;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::StoredMessage;
 use atlas_core::serialize::{OrderProtocolLog, OrderProtocolProof};
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::msg_log::deciding_log::CompletedBatch;
 
@@ -374,7 +374,11 @@ impl<O> Orderable for DecisionLog<O> {
     }
 }
 
-impl<O> OrderProtocolLog for DecisionLog<O> {}
+impl<O> OrderProtocolLog for DecisionLog<O> {
+    fn first_seq(&self) -> Option<SeqNo> {
+        self.proofs().first().map(|p| p.sequence_number())
+    }
+}
 
 impl<O> OrderProtocolProof for Proof<O> {}
 
diff --git a/febft-pbft-consensus/src/bft/msg_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/mod.rs
index 23841f79..8f1fd3cd 100644
--- a/febft-pbft-consensus/src/bft/msg_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/mod.rs
@@ -9,7 +9,7 @@ use atlas_common::node_id::NodeId;
 use atlas_common::ordering::SeqNo;
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::RequestMessage;
 use atlas_core::state_transfer::Checkpoint;
 
@@ -22,9 +22,9 @@ pub mod decisions;
 pub mod deciding_log;
 pub mod decided_log;
 
-pub fn initialize_decided_log<D: SharedData + 'static, PL>(node_id: NodeId,
-                                                       persistent_log: PL,
-                                                       state: Option<DecisionLog<D::Request>>) -> Result<Log<D, PL>> {
+pub fn initialize_decided_log<D: ApplicationData + 'static, PL>(node_id: NodeId,
+                                                                persistent_log: PL,
+                                                                state: Option<DecisionLog<D::Request>>) -> Result<Log<D, PL>> {
     Ok(Log::init_decided_log(node_id, persistent_log, state))
 }
 
diff --git a/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs
index 2ddf758e..ba219b35 100644
--- a/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs
@@ -4,20 +4,20 @@ use atlas_common::channel;
 use atlas_common::channel::{ChannelSyncRx, ChannelSyncTx};
 use atlas_communication::message::StoredMessage;
 use atlas_communication::Node;
-use atlas_execution::app::Service;
 use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{RequestMessage, StoredRequestMessage};
-use atlas_core::serialize::StateTransferMessage;
+use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use crate::bft::PBFT;
 
-pub type BatchType<D: SharedData> = Vec<StoredRequestMessage<D::Request>>;
+pub type BatchType<D: ApplicationData> = Vec<StoredRequestMessage<D::Request>>;
 
 ///TODO:
-pub struct FollowerProposer<D, ST, NT>
-    where D: SharedData + 'static,
+pub struct FollowerProposer<D, ST, LP, NT>
+    where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          NT: Node<PBFT<D, ST>> {
+          LP: LogTransferMessage + 'static,
+          NT: Node<PBFT<D, ST, LP>> {
     batch_channel: (ChannelSyncTx<BatchType<D>>, ChannelSyncRx<BatchType<D>>),
     //For request execution
     executor_handle: ExecutorHandle<D>,
@@ -30,7 +30,7 @@ pub struct FollowerProposer<D, ST, NT>
     target_global_batch_size: usize,
     //Time limit for generating a batch with target_global_batch_size size
     global_batch_time_limit: u128,
-    _phantom: PhantomData<ST>
+    _phantom: PhantomData<(ST, LP)>
 }
 
 
@@ -38,10 +38,11 @@ pub struct FollowerProposer<D, ST, NT>
 const BATCH_CHANNEL_SIZE: usize = 1024;
 
 
-impl<D, ST, NT> FollowerProposer<D, ST, NT>
-    where D: SharedData + 'static,
+impl<D, ST, LP, NT> FollowerProposer<D, ST, LP, NT>
+    where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          NT: Node<PBFT<D, ST>> {
+          LP: LogTransferMessage + 'static,
+          NT: Node<PBFT<D, ST, LP>> {
     pub fn new(
         node: Arc<NT>,
         executor: ExecutorHandle<D>,
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index 526248a6..bbc1d001 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -17,12 +17,12 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::threadpool;
 use atlas_communication::message::{NetworkMessage, NetworkMessageKind, StoredMessage};
 use atlas_communication::Node;
-use atlas_execution::app::{Request, Service, UnorderedBatch};
+use atlas_execution::app::{Request, UnorderedBatch};
 use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage, SystemMessage};
 use atlas_core::request_pre_processing::{BatchOutput, PreProcessorMessage, PreProcessorOutputMessage};
-use atlas_core::serialize::StateTransferMessage;
+use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
 use atlas_metrics::metrics::{metric_duration, metric_increment, metric_local_duration_end, metric_local_duration_start, metric_store_count};
 use crate::bft::config::ProposerConfig;
@@ -40,7 +40,7 @@ pub type BatchType<R> = Vec<StoredRequestMessage<R>>;
 ///as well as creating new batches and delivering them to the batch_channel
 ///Another thread will then take from this channel and propose the requests
 pub struct Proposer<D, NT>
-    where D: SharedData + 'static {
+    where D: ApplicationData + 'static {
     /// Channel for the reception of batches from the pre processing module
     batch_reception: BatchOutput<D::Request>,
     /// Network Node
@@ -65,12 +65,12 @@ pub struct Proposer<D, NT>
 const TIMEOUT: Duration = Duration::from_micros(10);
 const PRINT_INTERVAL: usize = 10000;
 
-struct ProposeBuilder<D> where D: SharedData {
+struct ProposeBuilder<D> where D: ApplicationData {
     currently_accumulated: Vec<StoredRequestMessage<D::Request>>,
     last_proposal: Instant,
 }
 
-impl<D> ProposeBuilder<D> where D: SharedData {
+impl<D> ProposeBuilder<D> where D: ApplicationData {
     pub fn new(target_size: usize) -> Self {
         Self { currently_accumulated: Vec::with_capacity(target_size), last_proposal: Instant::now() }
     }
@@ -79,7 +79,7 @@ impl<D> ProposeBuilder<D> where D: SharedData {
 ///The size of the batch channel
 const BATCH_CHANNEL_SIZE: usize = 128;
 
-impl<D, NT> Proposer<D, NT> where D: SharedData + 'static {
+impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
     pub fn new(
         node: Arc<NT>,
         batch_input: BatchOutput<D::Request>,
@@ -108,9 +108,10 @@ impl<D, NT> Proposer<D, NT> where D: SharedData + 'static {
     }
 
     ///Start this work
-    pub fn start<ST>(self: Arc<Self>) -> JoinHandle<()>
+    pub fn start<ST, LP>(self: Arc<Self>) -> JoinHandle<()>
         where ST: StateTransferMessage + 'static,
-              NT: Node<PBFT<D, ST>> + 'static {
+              LP: LogTransferMessage + 'static,
+              NT: Node<PBFT<D, ST, LP>> + 'static {
         std::thread::Builder::new()
             .name(format!("Proposer thread"))
             .spawn(move || {
@@ -220,7 +221,6 @@ impl<D, NT> Proposer<D, NT> where D: SharedData + 'static {
                                         digest_vec.push(ClientRqInfo::new(digest, message.header().from(), message.message().sequence_number(), message.message().session_id()));
                                     }
                                 }
-
                             }
                             PreProcessorOutputMessage::DeDupedUnorderedRequests(mut messages) => {
                                 unordered_propose.currently_accumulated.append(&mut messages);
@@ -263,11 +263,12 @@ impl<D, NT> Proposer<D, NT> where D: SharedData + 'static {
     /// Attempt to propose an unordered request batch
     /// Fails if the batch is not large enough or the timeout
     /// Has not yet occurred
-    fn propose_unordered<ST>(
+    fn propose_unordered<ST, LP>(
         &self,
         propose: &mut ProposeBuilder<D>,
     ) -> bool where ST: StateTransferMessage + 'static,
-            NT: Node<PBFT<D, ST>> {
+                    LP: LogTransferMessage + 'static,
+                    NT: Node<PBFT<D, ST, LP>> {
         if !propose.currently_accumulated.is_empty() {
             let current_batch_size = propose.currently_accumulated.len();
 
@@ -333,11 +334,12 @@ impl<D, NT> Proposer<D, NT> where D: SharedData + 'static {
 
     /// attempt to propose the ordered requests that we have collected
     /// Returns true if a batch was proposed
-    fn propose_ordered<ST>(&self,
+    fn propose_ordered<ST, LP>(&self,
                            is_leader: bool,
                            propose: &mut ProposeBuilder<D>, ) -> bool
         where ST: StateTransferMessage + 'static,
-              NT: Node<PBFT<D, ST>> {
+              LP: LogTransferMessage + 'static,
+              NT: Node<PBFT<D, ST, LP>> {
 
         //Now let's deal with ordered requests
         if is_leader {
@@ -389,14 +391,14 @@ impl<D, NT> Proposer<D, NT> where D: SharedData + 'static {
 
     /// Proposes a new batch.
     /// (Basically broadcasts it to all of the members)
-    fn propose<ST>(
+    fn propose<ST, LP>(
         &self,
         seq: SeqNo,
         view: &ViewInfo,
         mut currently_accumulated: Vec<StoredRequestMessage<D::Request>>,
     ) where ST: StateTransferMessage + 'static,
-            NT: Node<PBFT<D, ST>> {
-
+            LP: LogTransferMessage + 'static,
+            NT: Node<PBFT<D, ST, LP>> {
         let has_pending_messages = self.consensus_guard.has_pending_view_change_reqs();
 
         let is_view_change_empty = {
diff --git a/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
index 63bf2c10..512fb679 100644
--- a/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
@@ -3,17 +3,17 @@ use atlas_common::crypto::hash::Digest;
 use atlas_common::ordering::Orderable;
 use atlas_communication::message::StoredMessage;
 use atlas_core::messages::ClientRqInfo;
-use atlas_execution::app::{Request, Service};
-use atlas_execution::serialize::SharedData;
+use atlas_execution::app::{Request};
+use atlas_execution::serialize::ApplicationData;
 
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
 
-pub struct FollowerSynchronizer<D: SharedData> {
+pub struct FollowerSynchronizer<D: ApplicationData> {
     _phantom: PhantomData<D>
 }
 
-impl<D: SharedData + 'static> FollowerSynchronizer<D> {
+impl<D: ApplicationData + 'static> FollowerSynchronizer<D> {
 
     pub fn new() -> Self {
         Self { _phantom: Default::default() }
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 657b8ae9..a413e29f 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -26,13 +26,13 @@ use atlas_common::node_id::NodeId;
 use atlas_communication::message::{Header, NetworkMessageKind, StoredMessage, System, WireMessage};
 use atlas_communication::{Node, NodePK, serialize};
 use atlas_communication::serialize::Buf;
-use atlas_execution::app::{Reply, Request, Service, State};
-use atlas_execution::serialize::SharedData;
+use atlas_execution::app::{Reply, Request};
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, RequestMessage, StoredRequestMessage, SystemMessage};
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
-use atlas_core::serialize::StateTransferMessage;
+use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use crate::bft::consensus::Consensus;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, FwdConsensusMessage, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
@@ -354,7 +354,7 @@ pub enum SynchronizerPollStatus<O> {
 }
 
 ///A trait describing some of the necessary methods for the synchronizer
-pub trait AbstractSynchronizer<D: SharedData + 'static> {
+pub trait AbstractSynchronizer<D: ApplicationData + 'static> {
     /// Returns information regarding the current view, such as
     /// the number of faulty replicas the system can tolerate.
     fn view(&self) -> ViewInfo;
@@ -366,12 +366,12 @@ pub trait AbstractSynchronizer<D: SharedData + 'static> {
     fn queue(&self, header: Header, message: ViewChangeMessage<D::Request>);
 }
 
-type CollectsType<D: SharedData> = IntMap<StoredMessage<ViewChangeMessage<D::Request>>>;
+type CollectsType<D: ApplicationData> = IntMap<StoredMessage<ViewChangeMessage<D::Request>>>;
 
 ///The synchronizer for the SMR protocol
 /// This part of the protocol is responsible for handling the changing of views and
 /// for keeping track of any timed out client requests
-pub struct Synchronizer<D: SharedData> {
+pub struct Synchronizer<D: ApplicationData> {
     phase: Cell<ProtoPhase>,
     //Tbo queue, keeps track of the current view and keeps messages arriving in order
     tbo: Mutex<TboQueue<D::Request>>,
@@ -391,9 +391,9 @@ pub struct Synchronizer<D: SharedData> {
 /// So we protect collects, watching and tbo as those are the fields that are going to be
 /// accessed by both those threads.
 /// Since the other fields are going to be accessed by just 1 thread, we just need them to be Send, which they are
-unsafe impl<D: SharedData> Sync for Synchronizer<D> {}
+unsafe impl<D: ApplicationData> Sync for Synchronizer<D> {}
 
-impl<D: SharedData + 'static> AbstractSynchronizer<D> for Synchronizer<D> {
+impl<D: ApplicationData + 'static> AbstractSynchronizer<D> for Synchronizer<D> {
     /// Returns some information regarding the current view, such as
     /// the number of faulty replicas the system can tolerate.
     fn view(&self) -> ViewInfo {
@@ -419,7 +419,7 @@ impl<D: SharedData + 'static> AbstractSynchronizer<D> for Synchronizer<D> {
 
 impl<D> Synchronizer<D>
     where
-        D: SharedData + 'static
+        D: ApplicationData + 'static
 {
     pub fn new_follower(view: ViewInfo) -> Arc<Self> {
         Arc::new(Self {
@@ -497,18 +497,19 @@ impl<D> Synchronizer<D>
     /// Advances the state of the view change state machine.
     //
     // TODO: retransmit STOP msgs
-    pub fn process_message<ST, NT, PL>(
+    pub fn process_message<ST, LP, NT, PL>(
         &self,
         header: Header,
         message: ViewChangeMessage<D::Request>,
         timeouts: &Timeouts,
         log: &mut Log<D, PL>,
         rq_pre_processor: &RequestPreProcessor<D::Request>,
-        consensus: &mut Consensus<D, ST, PL>,
+        consensus: &mut Consensus<D, ST, LP, PL>,
         node: &NT,
     ) -> SynchronizerStatus<D::Request>
-        where ST: StateTransferMessage,
-              NT: Node<PBFT<D, ST>>,
+        where ST: StateTransferMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: Node<PBFT<D, ST, LP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         debug!("{:?} // Processing view change message {:?} in phase {:?} from {:?}",
@@ -834,7 +835,7 @@ impl<D> Synchronizer<D>
 
                                 let forged_pre_prepare = NetworkMessageKind::from(forged_pre_prepare);
 
-                                let digest = serialize::serialize_digest::<Vec<u8>, PBFT<D, ST>>(
+                                let digest = serialize::serialize_digest::<Vec<u8>, PBFT<D, ST, LP>>(
                                     &forged_pre_prepare,
                                     &mut buf,
                                 ).unwrap();
@@ -966,9 +967,9 @@ impl<D> Synchronizer<D>
 
                 // leader has already performed this computation in the
                 // STOP-DATA phase of Mod-SMaRt
-                let signed: Vec<_> = signed_collects::<D, _, _>(node, collects);
+                let signed: Vec<_> = signed_collects::<D, _, _, _>(node, collects);
 
-                let proof = highest_proof::<D, _, _, _>(&current_view, node, signed.iter());
+                let proof = highest_proof::<D, _, _, _, _>(&current_view, node, signed.iter());
 
                 let curr_cid = proof
                     .map(|p| p.sequence_number())
@@ -1016,14 +1017,16 @@ impl<D> Synchronizer<D>
     }
 
     /// Resume the view change protocol after running the CST protocol.
-    pub fn resume_view_change<ST, NT, PL>(
+    pub fn resume_view_change<ST, LP, NT, PL>(
         &self,
         log: &mut Log<D, PL>,
         timeouts: &Timeouts,
-        consensus: &mut Consensus<D, ST, PL>,
+        consensus: &mut Consensus<D, ST, LP, PL>,
         node: &NT,
     ) -> Option<()>
-        where ST: StateTransferMessage, NT: Node<PBFT<D, ST>>,
+        where ST: StateTransferMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: Node<PBFT<D, ST, LP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         let state = self.finalize_state.borrow_mut().take()?;
@@ -1051,7 +1054,7 @@ impl<D> Synchronizer<D>
     /// that have timed out on the current replica.
     /// If the timed out requests are None, that means that the view change
     /// originated in the other replicas.
-    pub fn begin_view_change<ST, NT, PL>(
+    pub fn begin_view_change<ST, LP, NT, PL>(
         &self,
         timed_out: Option<Vec<StoredRequestMessage<D::Request>>>,
         node: &NT,
@@ -1059,7 +1062,8 @@ impl<D> Synchronizer<D>
         _log: &Log<D, PL>,
     )
         where ST: StateTransferMessage + 'static,
-              NT: Node<PBFT<D, ST>>,
+              LP: LogTransferMessage + 'static,
+              NT: Node<PBFT<D, ST, LP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         match (self.phase.get(), &timed_out) {
@@ -1105,7 +1109,7 @@ impl<D> Synchronizer<D>
 
     // this function mostly serves the purpose of consuming
     // values with immutable references, to allow borrowing data mutably
-    fn pre_finalize< PL>(
+    fn pre_finalize<PL>(
         &self,
         state: FinalizeState<D::Request>,
         proof: Option<&Proof<D::Request>>,
@@ -1138,16 +1142,17 @@ impl<D> Synchronizer<D>
 
     /// Finalize a view change and install the new view in the other
     /// state machines (Consensus)
-    fn finalize<ST, NT, PL>(
+    fn finalize<ST, LP, NT, PL>(
         &self,
         state: FinalizeState<D::Request>,
         log: &mut Log<D, PL>,
         timeouts: &Timeouts,
-        consensus: &mut Consensus<D, ST, PL>,
+        consensus: &mut Consensus<D, ST, LP, PL>,
         node: &NT,
     ) -> SynchronizerStatus<D::Request>
-        where ST: StateTransferMessage,
-              NT: Node<PBFT<D, ST>>,
+        where ST: StateTransferMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: Node<PBFT<D, ST, LP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         let FinalizeState {
@@ -1232,10 +1237,12 @@ impl<D> Synchronizer<D>
     /// Forward the requests that have timed out to the whole network
     /// So that everyone knows about (including a leader that could still be correct, but
     /// Has not received the requests from the client)
-    pub fn forward_requests<ST, NT>(&self,
-                                    timed_out: Vec<StoredRequestMessage<D::Request>>,
-                                    node: &NT)
-        where ST: StateTransferMessage + 'static, NT: Node<PBFT<D, ST>> {
+    pub fn forward_requests<ST, LP, NT>(&self,
+                                        timed_out: Vec<StoredRequestMessage<D::Request>>,
+                                        node: &NT)
+        where ST: StateTransferMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: Node<PBFT<D, ST, LP>> {
         match &self.accessory {
             SynchronizerAccessory::Follower(_) => {}
             SynchronizerAccessory::Replica(rep) => {
@@ -1280,20 +1287,22 @@ impl<D> Synchronizer<D>
 
     // TODO: quorum sizes may differ when we implement reconfiguration
     #[inline]
-    fn highest_proof<'a, ST, NT>(
+    fn highest_proof<'a, ST, LP, NT>(
         guard: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request>>>,
         view: &ViewInfo,
         node: &NT,
     ) -> Option<&'a Proof<D::Request>>
-        where ST: StateTransferMessage + 'static, NT: Node<PBFT<D, ST>>
+        where ST: StateTransferMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: Node<PBFT<D, ST, LP>>
     {
-        highest_proof::<D, _, _, _>(&view, node, guard.values())
+        highest_proof::<D, _, _, _, _>(&view, node, guard.values())
     }
 }
 
 ///The accessory services that complement the base follower state machine
 /// This allows us to maximize code re usage and therefore reduce the amount of failure places
-pub enum SynchronizerAccessory<D: SharedData> {
+pub enum SynchronizerAccessory<D: ApplicationData> {
     Follower(FollowerSynchronizer<D>),
     Replica(ReplicaSynchronizer<D>),
 }
@@ -1512,26 +1521,28 @@ fn normalized_collects<'a, O: 'a>(
     })
 }
 
-fn signed_collects<D, ST, NT>(
+fn signed_collects<D, ST, LP, NT>(
     node: &NT,
     collects: Vec<StoredMessage<ViewChangeMessage<D::Request>>>,
 ) -> Vec<StoredMessage<ViewChangeMessage<D::Request>>>
     where
-        D: SharedData + 'static,
+        D: ApplicationData + 'static,
         ST: StateTransferMessage + 'static,
-        NT: Node<PBFT<D, ST>>
+        LP: LogTransferMessage + 'static,
+        NT: Node<PBFT<D, ST, LP>>
 {
     collects
         .into_iter()
-        .filter(|stored| validate_signature::<D, _, _, _>(node, stored))
+        .filter(|stored| validate_signature::<D, _, _, _, _>(node, stored))
         .collect()
 }
 
-fn validate_signature<'a, D, M, ST, NT>(node: &'a NT, stored: &'a StoredMessage<M>) -> bool
+fn validate_signature<'a, D, M, ST, LP, NT>(node: &'a NT, stored: &'a StoredMessage<M>) -> bool
     where
-        D: SharedData + 'static,
+        D: ApplicationData + 'static,
         ST: StateTransferMessage + 'static,
-        NT: Node<PBFT<D, ST>>
+        LP: LogTransferMessage + 'static,
+        NT: Node<PBFT<D, ST, LP>>
 {
     //TODO: Fix this as I believe it will always be false
     let wm = match WireMessage::from_header(*stored.header()) {
@@ -1557,16 +1568,17 @@ fn validate_signature<'a, D, M, ST, NT>(node: &'a NT, stored: &'a StoredMessage<
     wm.is_valid(Some(&key), false)
 }
 
-fn highest_proof<'a, D, I, ST, NT>(
+fn highest_proof<'a, D, I, ST, LP, NT>(
     view: &ViewInfo,
     node: &NT,
     collects: I,
 ) -> Option<&'a Proof<D::Request>>
     where
-        D: SharedData + 'static,
+        D: ApplicationData + 'static,
         I: Iterator<Item=&'a StoredMessage<ViewChangeMessage<D::Request>>>,
         ST: StateTransferMessage + 'static,
-        NT: Node<PBFT<D, ST>>
+        LP: LogTransferMessage + 'static,
+        NT: Node<PBFT<D, ST, LP>>
 {
     collect_data(collects)
         // fetch proofs
@@ -1587,7 +1599,7 @@ fn highest_proof<'a, D, I, ST, NT>(
                         .unwrap_or(false)
                 })
                 .filter(move |&stored|
-                    { validate_signature::<D, _, _, _>(node, stored) })
+                    { validate_signature::<D, _, _, _, _>(node, stored) })
                 .count() >= view.params().quorum();
 
             let prepares_valid = proof
@@ -1602,7 +1614,7 @@ fn highest_proof<'a, D, I, ST, NT>(
                         .unwrap_or(false)
                 })
                 .filter(move |&stored|
-                    { validate_signature::<D, _, _, _>(node, stored) })
+                    { validate_signature::<D, _, _, _, _>(node, stored) })
                 .count() >= view.params().quorum();
 
             debug!("{:?} // Proof {:?} is valid? commits valid: {:?} &&  prepares_valid: {:?}",
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 15b8e0a9..8642bfee 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -18,12 +18,12 @@ use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{NetworkMessageKind, StoredMessage, System};
 use atlas_communication::{Node};
-use atlas_execution::app::{Request, Service};
-use atlas_execution::serialize::SharedData;
+use atlas_execution::app::{Request};
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, RequestMessage, StoredRequestMessage, SystemMessage};
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
-use atlas_core::serialize::StateTransferMessage;
+use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, TimeoutPhase, Timeouts};
 use atlas_metrics::metrics::{metric_duration, metric_increment};
 use crate::bft::consensus::Consensus;
@@ -43,12 +43,12 @@ use super::{AbstractSynchronizer, Synchronizer, SynchronizerStatus};
 // - TboQueue for sync phase messages
 // This synchronizer will only move forward on replica messages
 
-pub struct ReplicaSynchronizer<D: SharedData> {
+pub struct ReplicaSynchronizer<D: ApplicationData> {
     timeout_dur: Cell<Duration>,
     _phantom: PhantomData<D>,
 }
 
-impl<D: SharedData + 'static> ReplicaSynchronizer<D> {
+impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     pub fn new(timeout_dur: Duration) -> Self {
         Self {
             timeout_dur: Cell::new(timeout_dur),
@@ -63,17 +63,19 @@ impl<D: SharedData + 'static> ReplicaSynchronizer<D> {
     ///
     /// Therefore, we start by clearing our stopped requests and treating them as
     /// newly proposed requests (by resetting their timer)
-    pub(super) fn handle_stopping_quorum<ST, NT, PL>(
+    pub(super) fn handle_stopping_quorum<ST, LP, NT, PL>(
         &self,
         base_sync: &Synchronizer<D>,
         previous_view: ViewInfo,
-        consensus: &Consensus<D, ST, PL>,
+        consensus: &Consensus<D, ST, LP, PL>,
         log: &Log<D, PL>,
         pre_processor: &RequestPreProcessor<D::Request>,
         timeouts: &Timeouts,
         node: &NT,
     )
-        where ST: StateTransferMessage + 'static, NT: Node<PBFT<D, ST>>,
+        where ST: StateTransferMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: Node<PBFT<D, ST, LP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>> {
         // NOTE:
         // - install new view (i.e. update view seq no) (Done in the synchronizer)
@@ -113,13 +115,15 @@ impl<D: SharedData + 'static> ReplicaSynchronizer<D> {
     /// Start a new view change
     /// Receives the requests that it should send to the other
     /// nodes in its STOP message
-    pub(super) fn handle_begin_view_change<ST, NT>(
+    pub(super) fn handle_begin_view_change<ST, LP, NT>(
         &self,
         base_sync: &Synchronizer<D>,
         timeouts: &Timeouts,
         node: &NT,
         timed_out: Option<Vec<StoredRequestMessage<D::Request>>>,
-    ) where ST: StateTransferMessage + 'static, NT: Node<PBFT<D, ST>> {
+    ) where ST: StateTransferMessage + 'static,
+            LP: LogTransferMessage + 'static,
+            NT: Node<PBFT<D, ST, LP>> {
         // stop all timers
         self.unwatch_all_requests(timeouts);
 
@@ -332,12 +336,14 @@ impl<D: SharedData + 'static> ReplicaSynchronizer<D> {
 
     /// Forward the requests that timed out, `timed_out`, to all the nodes in the
     /// current view.
-    pub fn forward_requests<ST, NT>(
+    pub fn forward_requests<ST, LP, NT>(
         &self,
         base_sync: &Synchronizer<D>,
         timed_out: Vec<StoredRequestMessage<D::Request>>,
         node: &NT,
-    ) where ST: StateTransferMessage + 'static, NT: Node<PBFT<D, ST>> {
+    ) where ST: StateTransferMessage + 'static,
+            LP: LogTransferMessage + 'static,
+            NT: Node<PBFT<D, ST, LP>> {
         let message = SystemMessage::ForwardedRequestMessage(ForwardedRequestsMessage::new(timed_out));
         let targets = NodeId::targets(0..base_sync.view().params().n());
         node.broadcast(NetworkMessageKind::from(message), targets);
@@ -432,4 +438,4 @@ impl<D: SharedData + 'static> ReplicaSynchronizer<D> {
 /// So we protect collects, watching and tbo as those are the fields that are going to be
 /// accessed by both those threads.
 /// Since the other fields are going to be accessed by just 1 thread, we just need them to be Send, which they are
-unsafe impl<D: SharedData> Sync for ReplicaSynchronizer<D> {}
+unsafe impl<D: ApplicationData> Sync for ReplicaSynchronizer<D> {}
diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 358f7676..04ed1e4e 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -21,7 +21,7 @@ use atlas_communication::Node;
 use atlas_communication::message::{Header, NetworkMessageKind, StoredMessage};
 use atlas_execution::app::{Application, Reply, Request, Service, State};
 use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{StateTransfer, SystemMessage};
 use atlas_core::ordering_protocol::{ExecutionResult, OrderingProtocol, SerProof, View};
 use atlas_core::persistent_log::{MonolithicStateLog, PersistableStateTransferProtocol, StateTransferProtocolLog, WriteMode};
@@ -228,7 +228,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
     type Serialization = CSTMsg<S>;
 
     fn request_latest_state<D, OP, LP, V>(&mut self, view: V) -> Result<()>
-        where D: SharedData + 'static,
+        where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
@@ -240,7 +240,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
 
     fn handle_off_ctx_message<D, OP, LP, V>(&mut self, view: V, message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
                                             -> Result<()>
-        where D: SharedData + 'static,
+        where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
@@ -285,7 +285,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
                                      view: V,
                                      message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
                                      -> Result<STResult>
-        where D: SharedData + 'static,
+        where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
@@ -351,7 +351,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
     }
 
     fn handle_app_state_requested<D, OP, LP, V>(&mut self, view: V, seq: SeqNo) -> Result<ExecutionResult>
-        where D: SharedData + 'static,
+        where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>, V: NetworkView {
@@ -378,7 +378,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
     }
 
     fn handle_timeout<D, OP, LP, V>(&mut self, view: V, timeout: Vec<RqTimeout>) -> Result<STTimeoutResult>
-        where D: SharedData + 'static,
+        where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
@@ -412,7 +412,7 @@ impl<S, NT, PL> MonolithicStateTransfer<S, NT, PL> for CollabStateTransfer<S, NT
     }
 
     fn handle_state_received_from_app<D, OP, LP, V>(&mut self, view: V, state: Arc<ReadOnly<Checkpoint<S>>>) -> Result<()>
-        where D: SharedData + 'static,
+        where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
@@ -467,7 +467,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         &mut self,
         header: Header,
         message: CstMessage<S>)
-        where D: SharedData + 'static,
+        where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>
@@ -500,7 +500,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     /// Process the entire list of pending state transfer requests
     /// This will only reply to the latest request sent by each of the replicas
     fn process_pending_state_requests<D, OP, LP>(&mut self)
-        where D: SharedData + 'static,
+        where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>> {
@@ -537,7 +537,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         &mut self,
         header: Header,
         message: CstMessage<S>,
-    ) where D: SharedData + 'static,
+    ) where D: ApplicationData + 'static,
             OP: OrderingProtocolMessage,
             LP: LogTransferMessage,
             NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>
@@ -589,7 +589,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         view: V,
         progress: CstProgress<S>,
     ) -> CstStatus<S>
-        where D: SharedData + 'static,
+        where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
@@ -848,7 +848,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     /// Returns a bool to signify if we must move to the Retrieving state
     /// If the timeout is no longer relevant, returns false (Can remain in current phase)
     pub fn cst_request_timed_out<D, OP, LP, V>(&mut self, seq: SeqNo, view: V) -> bool
-        where D: SharedData + 'static,
+        where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>, V: NetworkView {
@@ -905,7 +905,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     pub fn request_latest_consensus_seq_no<D, OP, LP, V>(
         &mut self,
         view: V,
-    ) where D: SharedData + 'static,
+    ) where D: ApplicationData + 'static,
             OP: OrderingProtocolMessage,
             LP: LogTransferMessage,
             NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
@@ -939,7 +939,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     /// Used by a recovering node to retrieve the latest state.
     pub fn request_latest_state<D, OP, LP, V>(
         &mut self, view: V,
-    ) where D: SharedData + 'static,
+    ) where D: ApplicationData + 'static,
             OP: OrderingProtocolMessage,
             LP: LogTransferMessage,
             NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
diff --git a/febft-state-transfer/src/message/serialize/capnp/mod.rs b/febft-state-transfer/src/message/serialize/capnp/mod.rs
index 2619e68f..366799e9 100644
--- a/febft-state-transfer/src/message/serialize/capnp/mod.rs
+++ b/febft-state-transfer/src/message/serialize/capnp/mod.rs
@@ -1,16 +1,16 @@
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_common::error::*;
 use atlas_core::state_transfer::StatefulOrderProtocol;
 use crate::message::CstMessage;
 
 fn serialize_state_transfer<D, SOP, NT>(mut state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Builder,
                                     msg: &CstMessage<D::State, SOP::DecLog, SOP::ViewInfo>) -> Result<()>
-    where D: SharedData, SOP: StatefulOrderProtocol<D, NT> {
+    where D: ApplicationData, SOP: StatefulOrderProtocol<D, NT> {
     Ok(())
 }
 
 fn deserialize_state_transfer<D, SOP, NT>(state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Reader)
                                       -> Result<CstMessage<D::State, SOP::DecLog, SOP::ViewInfo>>
-    where D: SharedData, SOP: StatefulOrderProtocol<D, NT> {
+    where D: ApplicationData, SOP: StatefulOrderProtocol<D, NT> {
     Err(Error::simple(ErrorKind::CommunicationSerialize))
 }
diff --git a/febft-state-transfer/src/message/serialize/mod.rs b/febft-state-transfer/src/message/serialize/mod.rs
index 542e0536..e9f8b814 100644
--- a/febft-state-transfer/src/message/serialize/mod.rs
+++ b/febft-state-transfer/src/message/serialize/mod.rs
@@ -3,7 +3,7 @@ mod capnp;
 
 use std::marker::PhantomData;
 use std::time::Instant;
-use atlas_execution::serialize::SharedData;
+use atlas_execution::serialize::ApplicationData;
 use atlas_core::serialize::{OrderingProtocolMessage, StatefulOrderProtocolMessage, StateTransferMessage};
 use atlas_core::state_transfer::{StatefulOrderProtocol, StateTransferProtocol};
 use crate::CollabStateTransfer;

From 629bc1d0f9172c33d9d9955dbbc7c9748e787797 Mon Sep 17 00:00:00 2001
From: nuno1212s <nuno.neto.g@gmail.com>
Date: Mon, 19 Jun 2023 21:55:43 +0100
Subject: [PATCH 05/80] A lot more fixed compilation issues.

---
 febft-state-transfer/src/lib.rs                   | 13 +++++++------
 febft-state-transfer/src/message/serialize/mod.rs |  2 +-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 04ed1e4e..611cd550 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -19,12 +19,12 @@ use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::Node;
 use atlas_communication::message::{Header, NetworkMessageKind, StoredMessage};
-use atlas_execution::app::{Application, Reply, Request, Service, State};
+use atlas_execution::app::{Application, Reply, Request};
 use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{StateTransfer, SystemMessage};
 use atlas_core::ordering_protocol::{ExecutionResult, OrderingProtocol, SerProof, View};
-use atlas_core::persistent_log::{MonolithicStateLog, PersistableStateTransferProtocol, StateTransferProtocolLog, WriteMode};
+use atlas_core::persistent_log::{MonolithicStateLog, PersistableStateTransferProtocol, WriteMode};
 use atlas_core::serialize::{LogTransferMessage, NetworkView, OrderingProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage, StateTransferMessage};
 use atlas_core::state_transfer::{Checkpoint, CstM, StateTransferProtocol, STResult, STTimeoutResult};
 use atlas_core::state_transfer::monolithic_state::MonolithicStateTransfer;
@@ -329,7 +329,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
 
                     self.request_latest_state(view);
                 } else {
-                    debug!("{:?} // Not installing sequence number nor requesting state ???? {:?} {:?}", self.node.id(), order_protocol.sequence_number(), seq);
+                    debug!("{:?} // Not installing sequence number nor requesting state ???? {:?} {:?}", self.node.id(), self.current_checkpoint_state.sequence_number(), seq);
                     return Ok(STResult::StateTransferNotNeeded(seq));
                 }
             }
@@ -484,14 +484,14 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
             }
         };
 
-        let kind = CstMessageKind::ReplyStateCid(seq);
+        let kind = CstMessageKind::ReplyStateCid(seq.clone());
 
         let reply = CstMessage::new(message.sequence_number(), kind);
 
         let network_msg = NetworkMessageKind::from(SystemMessage::from_state_transfer_message(reply));
 
         debug!("{:?} // Replying to {:?} seq {:?} with seq no {:?}", self.node.id(),
-            header.from(), message.sequence_number(), order_protocol.sequence_number());
+            header.from(), message.sequence_number(), seq);
 
         self.node.send(network_msg, header.from(), true);
     }
@@ -644,7 +644,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                     CstMessageKind::ReplyStateCid(state_cid) => {
                         todo!();
 
-                        debug!("{:?} // Received CID vote {:?} from {:?}", self.node.id(), seq, header.from());
+                        /*debug!("{:?} // Received CID vote {:?} from {:?}", self.node.id(), seq, header.from());
 
                         match seq.cmp(&self.largest_cid) {
                             Ordering::Greater => {
@@ -656,6 +656,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                             }
                             Ordering::Less => (),
                         }
+                         */
                     }
                     CstMessageKind::RequestLatestConsensusSeq => {
                         self.process_request_seq(header, message);
diff --git a/febft-state-transfer/src/message/serialize/mod.rs b/febft-state-transfer/src/message/serialize/mod.rs
index e9f8b814..5fe7b60f 100644
--- a/febft-state-transfer/src/message/serialize/mod.rs
+++ b/febft-state-transfer/src/message/serialize/mod.rs
@@ -5,7 +5,7 @@ use std::marker::PhantomData;
 use std::time::Instant;
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::serialize::{OrderingProtocolMessage, StatefulOrderProtocolMessage, StateTransferMessage};
-use atlas_core::state_transfer::{StatefulOrderProtocol, StateTransferProtocol};
+use atlas_core::state_transfer::{StateTransferProtocol};
 use crate::CollabStateTransfer;
 use crate::message::CstMessage;
 

From 2d95b2f15bf1c75ac6fedb01b77569b2230be3e3 Mon Sep 17 00:00:00 2001
From: nuno1212s <nuno.neto.g@gmail.com>
Date: Tue, 20 Jun 2023 13:40:15 +0100
Subject: [PATCH 06/80] Fixed compile issues with replicas. Now just missing
 the state transfer.

---
 febft-state-transfer/src/lib.rs | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 611cd550..7ae83d09 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -220,8 +220,6 @@ macro_rules! getmessage {
     }};
 }
 
-type Serialization<S, OP, NT> = <S as StateTransferProtocol<S, OP, NT>>::Serialization;
-
 impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT, PL>
     where S: MonolithicState + 'static,
 {
@@ -431,6 +429,8 @@ impl<S, NT, PL> MonolithicStateTransfer<S, NT, PL> for CollabStateTransfer<S, NT
     }
 }
 
+type Ser<ST: StateTransferProtocol<S, NT, PL>, S, NT, PL> = <ST as StateTransferProtocol<S, NT, PL>>::Serialization;
+
 // TODO: request timeouts
 impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     where
@@ -470,7 +470,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
-              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>
+              NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>>
     {
         let seq = match &self.current_checkpoint_state {
             CheckpointState::PartialWithEarlier { seq, earlier, } => {
@@ -503,7 +503,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
-              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>> {
+              NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>> {
         let waiting = std::mem::replace(&mut self.phase, ProtoPhase::Init);
 
         if let ProtoPhase::WaitingCheckpoint(reqs) = waiting {
@@ -540,7 +540,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     ) where D: ApplicationData + 'static,
             OP: OrderingProtocolMessage,
             LP: LogTransferMessage,
-            NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>
+            NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>>
     {
         match &mut self.phase {
             ProtoPhase::Init => {}
@@ -592,7 +592,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
-              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>>,
               V: NetworkView
     {
         match self.phase {
@@ -852,7 +852,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage,
               LP: LogTransferMessage,
-              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>, V: NetworkView {
+              NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>>, V: NetworkView {
         let status = self.timed_out(seq);
 
         match status {
@@ -909,7 +909,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     ) where D: ApplicationData + 'static,
             OP: OrderingProtocolMessage,
             LP: LogTransferMessage,
-            NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+            NT: Node<ServiceMsg<D, OP,Ser<Self, S, OP, NT>, LP>>,
             V: NetworkView
     {
 
@@ -943,7 +943,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     ) where D: ApplicationData + 'static,
             OP: OrderingProtocolMessage,
             LP: LogTransferMessage,
-            NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+            NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>>,
             V: NetworkView
     {
         // reset hashmap of received states

From 5e5fc9d28560565fc862b7d50c21cc634c6aa5d3 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Tue, 20 Jun 2023 21:14:03 +0100
Subject: [PATCH 07/80] Fixed state transfer protocol compilation, still have
 to fix the actual protocol functioning.

---
 febft-state-transfer/src/lib.rs               | 107 +++++++++---------
 .../src/message/serialize/mod.rs              |   5 +-
 2 files changed, 57 insertions(+), 55 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 7ae83d09..c53a255e 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -222,13 +222,14 @@ macro_rules! getmessage {
 
 impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT, PL>
     where S: MonolithicState + 'static,
+          PL: MonolithicStateLog<S> + 'static
 {
     type Serialization = CSTMsg<S>;
 
     fn request_latest_state<D, OP, LP, V>(&mut self, view: V) -> Result<()>
         where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage,
-              LP: LogTransferMessage,
+              OP: OrderingProtocolMessage + 'static,
+              LP: LogTransferMessage + 'static,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
               V: NetworkView {
         self.request_latest_consensus_seq_no::<D, OP, LP, V>(view);
@@ -239,8 +240,8 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
     fn handle_off_ctx_message<D, OP, LP, V>(&mut self, view: V, message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
                                             -> Result<()>
         where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage,
-              LP: LogTransferMessage,
+              OP: OrderingProtocolMessage + 'static,
+              LP: LogTransferMessage + 'static,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
               V: NetworkView {
         let (header, message) = message.into_inner();
@@ -250,7 +251,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         debug!("{:?} // Off context Message {:?} from {:?} with seq {:?}", self.node.id(), message, header.from(), message.sequence_number());
 
         match &message.kind() {
-            CstMessageKind::RequestLatestConsensusSeq => {
+            CstMessageKind::RequestStateCid => {
                 self.process_request_seq(header, message);
 
                 return Ok(());
@@ -270,7 +271,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
 
         match status {
             CstStatus::Nil => (),
-            // should not happen...
+// should not happen...
             _ => {
                 return Err("Invalid state reached!").wrapped(ErrorKind::CoreServer);
             }
@@ -284,8 +285,8 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
                                      message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
                                      -> Result<STResult>
         where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage,
-              LP: LogTransferMessage,
+              OP: OrderingProtocolMessage + 'static,
+              LP: LogTransferMessage + 'static,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
               V: NetworkView {
         let (header, message) = message.into_inner();
@@ -295,7 +296,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         debug!("{:?} // Message {:?} from {:?} while in phase {:?}", self.node.id(), message, header.from(), self.phase);
 
         match &message.kind() {
-            CstMessageKind::RequestLatestConsensusSeq => {
+            CstMessageKind::RequestStateCid => {
                 self.process_request_seq(header, message);
 
                 return Ok(STResult::StateTransferRunning);
@@ -317,7 +318,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         match status {
             CstStatus::Running => (),
             CstStatus::State(state) => {
-                self.install_channel.send(InstallStateMessage::new(state.checkpoint.clone())).unwrap();
+                self.install_channel.send(InstallStateMessage::new(state.checkpoint.state().clone())).unwrap();
 
                 return Ok(STResult::StateTransferFinished(state.checkpoint.sequence_number()));
             }
@@ -338,10 +339,10 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
                 self.request_latest_state(view);
             }
             CstStatus::Nil => {
-                // No actions are required for the CST
-                // This can happen for example when we already received the a quorum of sequence number replies
-                // And therefore we are already in the Init phase and we are still receiving replies
-                // And have not yet processed the
+// No actions are required for the CST
+// This can happen for example when we already received the a quorum of sequence number replies
+// And therefore we are already in the Init phase and we are still receiving replies
+// And have not yet processed the
             }
         }
 
@@ -350,8 +351,8 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
 
     fn handle_app_state_requested<D, OP, LP, V>(&mut self, view: V, seq: SeqNo) -> Result<ExecutionResult>
         where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage,
-              LP: LogTransferMessage,
+              OP: OrderingProtocolMessage + 'static,
+              LP: LogTransferMessage + 'static,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>, V: NetworkView {
         let earlier = std::mem::replace(&mut self.current_checkpoint_state, CheckpointState::None);
 
@@ -360,9 +361,9 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
             CheckpointState::Complete(earlier) => {
                 CheckpointState::PartialWithEarlier { seq, earlier }
             }
-            // FIXME: this may not be an invalid state after all; we may just be generating
-            // checkpoints too fast for the execution layer to keep up, delivering the
-            // hash digests of the appstate
+// FIXME: this may not be an invalid state after all; we may just be generating
+// checkpoints too fast for the execution layer to keep up, delivering the
+// hash digests of the appstate
             _ => {
                 error!("Invalid checkpoint state detected");
 
@@ -377,8 +378,8 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
 
     fn handle_timeout<D, OP, LP, V>(&mut self, view: V, timeout: Vec<RqTimeout>) -> Result<STTimeoutResult>
         where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage,
-              LP: LogTransferMessage,
+              OP: OrderingProtocolMessage + 'static,
+              LP: LogTransferMessage + 'static,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
               V: NetworkView {
         for cst_seq in timeout {
@@ -395,7 +396,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
 
 impl<S, NT, PL> MonolithicStateTransfer<S, NT, PL> for CollabStateTransfer<S, NT, PL>
     where S: MonolithicState + 'static,
-          PL: MonolithicStateLog<S> {
+          PL: MonolithicStateLog<S> + 'static {
     type Config = StateTransferConfig;
 
     fn initialize(config: Self::Config, timeouts: Timeouts, node: Arc<NT>,
@@ -411,8 +412,8 @@ impl<S, NT, PL> MonolithicStateTransfer<S, NT, PL> for CollabStateTransfer<S, NT
 
     fn handle_state_received_from_app<D, OP, LP, V>(&mut self, view: V, state: Arc<ReadOnly<Checkpoint<S>>>) -> Result<()>
         where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage,
-              LP: LogTransferMessage,
+              OP: OrderingProtocolMessage + 'static,
+              LP: LogTransferMessage + 'static,
               NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
               V: NetworkView {
         self.finalize_checkpoint(state)?;
@@ -435,7 +436,7 @@ type Ser<ST: StateTransferProtocol<S, NT, PL>, S, NT, PL> = <ST as StateTransfer
 impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     where
         S: MonolithicState + 'static,
-        PL: MonolithicStateLog<S>,
+        PL: MonolithicStateLog<S> + 'static,
 {
     /// Create a new instance of `CollabStateTransfer`.
     pub fn new(node: Arc<NT>, base_timeout: Duration, timeouts: Timeouts, persistent_log: PL, install_channel: ChannelSyncTx<InstallStateMessage<S>>) -> Self {
@@ -468,9 +469,9 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         header: Header,
         message: CstMessage<S>)
         where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage,
-              LP: LogTransferMessage,
-              NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>>
+              OP: OrderingProtocolMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>
     {
         let seq = match &self.current_checkpoint_state {
             CheckpointState::PartialWithEarlier { seq, earlier, } => {
@@ -501,9 +502,9 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     /// This will only reply to the latest request sent by each of the replicas
     fn process_pending_state_requests<D, OP, LP>(&mut self)
         where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage,
-              LP: LogTransferMessage,
-              NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>> {
+              OP: OrderingProtocolMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>{
         let waiting = std::mem::replace(&mut self.phase, ProtoPhase::Init);
 
         if let ProtoPhase::WaitingCheckpoint(reqs) = waiting {
@@ -528,19 +529,19 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
             map.into_values().for_each(|req| {
                 let (header, message) = req.into_inner();
 
-                self.process_request_state(header, message);
+                self.process_request_state::<D, OP, LP>(header, message);
             });
         }
     }
 
-    fn process_request_state<D, OP, LP, V>(
+    fn process_request_state<D, OP, LP>(
         &mut self,
         header: Header,
         message: CstMessage<S>,
     ) where D: ApplicationData + 'static,
-            OP: OrderingProtocolMessage,
-            LP: LogTransferMessage,
-            NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>>
+            OP: OrderingProtocolMessage + 'static,
+            LP: LogTransferMessage + 'static,
+            NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>
     {
         match &mut self.phase {
             ProtoPhase::Init => {}
@@ -590,9 +591,9 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         progress: CstProgress<S>,
     ) -> CstStatus<S>
         where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage,
-              LP: LogTransferMessage,
-              NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>>,
+              OP: OrderingProtocolMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
               V: NetworkView
     {
         match self.phase {
@@ -605,7 +606,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                 let (header, message) = getmessage!(progress, CstStatus::Nil);
 
                 match message.kind() {
-                    CstMessageKind::RequestLatestConsensusSeq => {
+                    CstMessageKind::RequestStateCid => {
                         self.process_request_seq(header, message);
                     }
                     CstMessageKind::RequestState => {
@@ -658,7 +659,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                         }
                          */
                     }
-                    CstMessageKind::RequestLatestConsensusSeq => {
+                    CstMessageKind::RequestStateCid => {
                         self.process_request_seq(header, message);
 
                         return CstStatus::Running;
@@ -724,8 +725,8 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                 if self.received_states.contains_key(&state_digest) {
                     let current_state = self.received_states.get_mut(&state_digest).unwrap();
 
-                    let current_state_seq: SeqNo = current_state.state.log().sequence_number();
-                    let recv_state_seq: SeqNo = state.log().sequence_number();
+                    let current_state_seq: SeqNo = current_state.state.checkpoint().sequence_number();
+                    let recv_state_seq: SeqNo = state.checkpoint().sequence_number();
 
                     match recv_state_seq.cmp(&current_state_seq) {
                         Ordering::Less | Ordering::Equal => {
@@ -850,9 +851,9 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     /// If the timeout is no longer relevant, returns false (Can remain in current phase)
     pub fn cst_request_timed_out<D, OP, LP, V>(&mut self, seq: SeqNo, view: V) -> bool
         where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage,
-              LP: LogTransferMessage,
-              NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>>, V: NetworkView {
+              OP: OrderingProtocolMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>, V: NetworkView {
         let status = self.timed_out(seq);
 
         match status {
@@ -907,9 +908,9 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         &mut self,
         view: V,
     ) where D: ApplicationData + 'static,
-            OP: OrderingProtocolMessage,
-            LP: LogTransferMessage,
-            NT: Node<ServiceMsg<D, OP,Ser<Self, S, OP, NT>, LP>>,
+            OP: OrderingProtocolMessage + 'static,
+            LP: LogTransferMessage + 'static,
+            NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
             V: NetworkView
     {
 
@@ -929,7 +930,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
         let message = CstMessage::new(
             cst_seq,
-            CstMessageKind::RequestLatestConsensusSeq,
+            CstMessageKind::RequestStateCid,
         );
 
         let targets = NodeId::targets(0..view.n());
@@ -941,9 +942,9 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     pub fn request_latest_state<D, OP, LP, V>(
         &mut self, view: V,
     ) where D: ApplicationData + 'static,
-            OP: OrderingProtocolMessage,
-            LP: LogTransferMessage,
-            NT: Node<ServiceMsg<D, OP, Ser<Self, S, OP, NT>, LP>>,
+            OP: OrderingProtocolMessage + 'static,
+            LP: LogTransferMessage + 'static,
+            NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
             V: NetworkView
     {
         // reset hashmap of received states
diff --git a/febft-state-transfer/src/message/serialize/mod.rs b/febft-state-transfer/src/message/serialize/mod.rs
index 5fe7b60f..516d5917 100644
--- a/febft-state-transfer/src/message/serialize/mod.rs
+++ b/febft-state-transfer/src/message/serialize/mod.rs
@@ -6,12 +6,13 @@ use std::time::Instant;
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::serialize::{OrderingProtocolMessage, StatefulOrderProtocolMessage, StateTransferMessage};
 use atlas_core::state_transfer::{StateTransferProtocol};
+use atlas_execution::state::monolithic_state::MonolithicState;
 use crate::CollabStateTransfer;
 use crate::message::CstMessage;
 
-pub struct CSTMsg<S>(PhantomData<(S)>);
+pub struct CSTMsg<S: MonolithicState>(PhantomData<(S)>);
 
-impl<S> StateTransferMessage for CSTMsg<S> {
+impl<S: MonolithicState> StateTransferMessage for CSTMsg<S> {
 
     type StateTransferMessage = CstMessage<S>;
 

From 5dcceb2317272e03d81a3334d32b2a21d96c54c0 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 21 Jun 2023 13:08:32 +0100
Subject: [PATCH 08/80] We are now successfully compiling.

---
 febft-state-transfer/src/lib.rs | 34 ++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index c53a255e..b657bc20 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -116,6 +116,10 @@ struct ReceivedState<S> {
     state: RecoveryState<S>,
 }
 
+struct ReceivedStateCid {
+    cid: SeqNo,
+    count: usize,
+}
 
 // NOTE: in this module, we may use cid interchangeably with
 // consensus sequence number
@@ -125,9 +129,9 @@ struct ReceivedState<S> {
 /// Durable State Machine Replication», by A. Bessani et al.
 pub struct CollabStateTransfer<S, NT, PL>
     where S: MonolithicState + 'static {
+    curr_seq: SeqNo,
     current_checkpoint_state: CheckpointState<S>,
     largest_cid: SeqNo,
-    cst_seq: SeqNo,
     latest_cid_count: usize,
     base_timeout: Duration,
     curr_timeout: Duration,
@@ -137,6 +141,7 @@ pub struct CollabStateTransfer<S, NT, PL>
     //voted: HashSet<NodeId>,
     node: Arc<NT>,
     received_states: HashMap<Digest, ReceivedState<S>>,
+    received_state_ids: HashMap<Digest, ReceivedStateCid>,
     phase: ProtoPhase<S>,
 
     install_channel: ChannelSyncTx<InstallStateMessage<S>>,
@@ -447,10 +452,11 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
             timeouts,
             node,
             received_states: collections::hash_map(),
+            received_state_ids: collections::hash_map(),
             phase: ProtoPhase::Init,
             largest_cid: SeqNo::ZERO,
             latest_cid_count: 0,
-            cst_seq: SeqNo::ZERO,
+            curr_seq: SeqNo::ZERO,
             persistent_log,
             install_channel,
         }
@@ -626,11 +632,11 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                 let (header, message) = getmessage!(progress, CstStatus::RequestStateCid);
 
                 debug!("{:?} // Received Cid with {} responses from {:?} for CST Seq {:?} vs Ours {:?}", self.node.id(),
-                   i, header.from(), message.sequence_number(), self.cst_seq);
+                   i, header.from(), message.sequence_number(), self.curr_seq);
 
                 // drop cst messages with invalid seq no
-                if message.sequence_number() != self.cst_seq {
-                    debug!("{:?} // Wait what? {:?} {:?}", self.node.id(), self.cst_seq, message.sequence_number());
+                if message.sequence_number() != self.curr_seq {
+                    debug!("{:?} // Wait what? {:?} {:?}", self.node.id(), self.curr_seq, message.sequence_number());
                     // FIXME: how to handle old or newer messages?
                     // BFT-SMaRt simply ignores messages with a
                     // value of `queryID` different from the current
@@ -643,6 +649,12 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
                 match message.kind() {
                     CstMessageKind::ReplyStateCid(state_cid) => {
+
+                        if let Some((cid, digest)) = state_cid {
+
+                        } else {
+
+                        }
                         todo!();
 
                         /*debug!("{:?} // Received CID vote {:?} from {:?}", self.node.id(), seq, header.from());
@@ -681,7 +693,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
                 debug!("{:?} // Quorum count {}, i: {}, cst_seq {:?}. Current Latest Cid: {:?}. Current Latest Cid Count: {}",
                         self.node.id(), view.quorum(), i,
-                        self.cst_seq, self.largest_cid, self.latest_cid_count);
+                        self.curr_seq, self.largest_cid, self.latest_cid_count);
 
                 if i == view.quorum() {
                     self.phase = ProtoPhase::Init;
@@ -706,7 +718,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
             ProtoPhase::ReceivingState(i) => {
                 let (header, mut message) = getmessage!(progress, CstStatus::RequestState);
 
-                if message.sequence_number() != self.cst_seq {
+                if message.sequence_number() != self.curr_seq {
                     // NOTE: check comment above, on ProtoPhase::ReceivingCid
                     return CstStatus::Running;
                 }
@@ -837,13 +849,13 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     }
 
     fn curr_seq(&mut self) -> SeqNo {
-        self.cst_seq
+        self.curr_seq
     }
 
     fn next_seq(&mut self) -> SeqNo {
-        self.cst_seq = self.cst_seq.next();
+        self.curr_seq = self.curr_seq.next();
 
-        self.cst_seq
+        self.curr_seq
     }
 
     /// Handle a timeout received from the timeouts layer.
@@ -873,7 +885,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     }
 
     fn timed_out(&mut self, seq: SeqNo) -> CstStatus<S> {
-        if seq != self.cst_seq {
+        if seq != self.curr_seq {
             // the timeout we received is for a request
             // that has already completed, therefore we ignore it
             //

From 7cdfcfded180e1f81eecc9a91ae54a41044bebf7 Mon Sep 17 00:00:00 2001
From: nuno1212s <nuno.neto.g@gmail.com>
Date: Thu, 22 Jun 2023 12:10:41 +0100
Subject: [PATCH 09/80] Work on the persistent logging to make it handle the
 new states.

---
 febft-state-transfer/Cargo.toml               |  3 +-
 febft-state-transfer/src/lib.rs               | 38 +++++++++----------
 .../src/message/serialize/capnp/mod.rs        | 13 ++++---
 3 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/febft-state-transfer/Cargo.toml b/febft-state-transfer/Cargo.toml
index c354226d..850a7b87 100644
--- a/febft-state-transfer/Cargo.toml
+++ b/febft-state-transfer/Cargo.toml
@@ -22,5 +22,4 @@ atlas-capnp = { path = "../../Atlas/atlas-capnp", optional = true }
 atlas-execution = { path = "../../Atlas/atlas-execution" }
 atlas-common = { path = "../../Atlas/atlas-common" }
 atlas-communication = { path = "../../Atlas/atlas-communication" }
-atlas-core = { path = "../../Atlas/atlas-core" }
-atlas-persistent-log = { path = "../../Atlas/atlas-persistent-log" }
\ No newline at end of file
+atlas-core = { path = "../../Atlas/atlas-core" }
\ No newline at end of file
diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index b657bc20..11eecb2f 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -510,7 +510,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>{
+              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>> {
         let waiting = std::mem::replace(&mut self.phase, ProtoPhase::Init);
 
         if let ProtoPhase::WaitingCheckpoint(reqs) = waiting {
@@ -649,27 +649,27 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
                 match message.kind() {
                     CstMessageKind::ReplyStateCid(state_cid) => {
-
                         if let Some((cid, digest)) = state_cid {
-
-                        } else {
-
-                        }
-                        todo!();
-
-                        /*debug!("{:?} // Received CID vote {:?} from {:?}", self.node.id(), seq, header.from());
-
-                        match seq.cmp(&self.largest_cid) {
-                            Ordering::Greater => {
-                                self.largest_cid = seq;
-                                self.latest_cid_count = 1;
-                            }
-                            Ordering::Equal => {
-                                self.latest_cid_count += 1;
+                            let received_state_cid = self.received_state_ids.entry(digest.clone()).or_insert_with(|| {
+                                ReceivedStateCid {
+                                    cid: *cid,
+                                    count: 0,
+                                }
+                            });
+
+                            if *cid > received_state_cid.cid {
+                                info!("{:?} // Received newer state for old cid {:?} vs new cid {:?} with digest {:?}.",
+                                    self.node.id(), received_state_cid.cid, *cid, digest);
+
+                                received_state_cid.cid = *cid;
+                                received_state_cid.count = 1;
+                            } else if cid == received_state_cid.cid {
+                                info!("{:?} // Received matching state for cid {:?} with digest {:?}. Count {}",
+                                self.node.id(), received_state_cid.cid, digest, received_state_cid.count + 1);
+
+                                received_state_cid.count += 1;
                             }
-                            Ordering::Less => (),
                         }
-                         */
                     }
                     CstMessageKind::RequestStateCid => {
                         self.process_request_seq(header, message);
diff --git a/febft-state-transfer/src/message/serialize/capnp/mod.rs b/febft-state-transfer/src/message/serialize/capnp/mod.rs
index 366799e9..b2ba29fa 100644
--- a/febft-state-transfer/src/message/serialize/capnp/mod.rs
+++ b/febft-state-transfer/src/message/serialize/capnp/mod.rs
@@ -1,16 +1,17 @@
 use atlas_execution::serialize::ApplicationData;
 use atlas_common::error::*;
 use atlas_core::state_transfer::StatefulOrderProtocol;
+use atlas_execution::state::divisible_state::DivisibleState;
 use crate::message::CstMessage;
 
-fn serialize_state_transfer<D, SOP, NT>(mut state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Builder,
-                                    msg: &CstMessage<D::State, SOP::DecLog, SOP::ViewInfo>) -> Result<()>
-    where D: ApplicationData, SOP: StatefulOrderProtocol<D, NT> {
+fn serialize_state_transfer<S>(mut state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Builder,
+                               msg: &CstMessage<S>) -> Result<()>
+    where S: DivisibleState {
     Ok(())
 }
 
-fn deserialize_state_transfer<D, SOP, NT>(state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Reader)
-                                      -> Result<CstMessage<D::State, SOP::DecLog, SOP::ViewInfo>>
-    where D: ApplicationData, SOP: StatefulOrderProtocol<D, NT> {
+fn deserialize_state_transfer<S>(state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Reader)
+                                 -> Result<CstMessage<S>>
+    where S: DivisibleState {
     Err(Error::simple(ErrorKind::CommunicationSerialize))
 }

From d75c429f750ccf501f99522933f5aac31ca82d62 Mon Sep 17 00:00:00 2001
From: nuno1212s <nuno.neto.g@gmail.com>
Date: Thu, 22 Jun 2023 14:49:49 +0100
Subject: [PATCH 10/80] Added some metrics to state and log transfer.

---
 febft-state-transfer/Cargo.toml         |  3 ++-
 febft-state-transfer/src/lib.rs         | 17 ++++++++++++-----
 febft-state-transfer/src/metrics/mod.rs | 13 +++++++++++++
 3 files changed, 27 insertions(+), 6 deletions(-)
 create mode 100644 febft-state-transfer/src/metrics/mod.rs

diff --git a/febft-state-transfer/Cargo.toml b/febft-state-transfer/Cargo.toml
index 850a7b87..aca2a5cf 100644
--- a/febft-state-transfer/Cargo.toml
+++ b/febft-state-transfer/Cargo.toml
@@ -22,4 +22,5 @@ atlas-capnp = { path = "../../Atlas/atlas-capnp", optional = true }
 atlas-execution = { path = "../../Atlas/atlas-execution" }
 atlas-common = { path = "../../Atlas/atlas-common" }
 atlas-communication = { path = "../../Atlas/atlas-communication" }
-atlas-core = { path = "../../Atlas/atlas-core" }
\ No newline at end of file
+atlas-core = { path = "../../Atlas/atlas-core" }
+atlas-metrics = { path = "../../Atlas/atlas-metrics" }
\ No newline at end of file
diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 11eecb2f..5ceccf1f 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -3,7 +3,7 @@
 use std::cmp::Ordering;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use log::{debug, error, info};
 #[cfg(feature = "serialize_serde")]
@@ -30,13 +30,16 @@ use atlas_core::state_transfer::{Checkpoint, CstM, StateTransferProtocol, STResu
 use atlas_core::state_transfer::monolithic_state::MonolithicStateTransfer;
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, Timeouts};
 use atlas_execution::state::monolithic_state::{InstallStateMessage, MonolithicState};
+use atlas_metrics::metrics::metric_duration;
 
 use crate::config::StateTransferConfig;
 use crate::message::{CstMessage, CstMessageKind};
 use crate::message::serialize::CSTMsg;
+use crate::metrics::STATE_TRANSFER_STATE_INSTALL_CLONE_TIME_ID;
 
 pub mod message;
 pub mod config;
+pub mod metrics;
 
 /// The state of the checkpoint
 pub enum CheckpointState<D> {
@@ -323,8 +326,12 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         match status {
             CstStatus::Running => (),
             CstStatus::State(state) => {
+                let start = Instant::now();
+
                 self.install_channel.send(InstallStateMessage::new(state.checkpoint.state().clone())).unwrap();
 
+                metric_duration(STATE_TRANSFER_STATE_INSTALL_CLONE_TIME_ID, start.elapsed());
+
                 return Ok(STResult::StateTransferFinished(state.checkpoint.sequence_number()));
             }
             CstStatus::SeqNo(seq) => {
@@ -344,10 +351,10 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
                 self.request_latest_state(view);
             }
             CstStatus::Nil => {
-// No actions are required for the CST
-// This can happen for example when we already received the a quorum of sequence number replies
-// And therefore we are already in the Init phase and we are still receiving replies
-// And have not yet processed the
+                // No actions are required for the CST
+                // This can happen for example when we already received the a quorum of sequence number replies
+                // And therefore we are already in the Init phase and we are still receiving replies
+                // And have not yet processed the
             }
         }
 
diff --git a/febft-state-transfer/src/metrics/mod.rs b/febft-state-transfer/src/metrics/mod.rs
new file mode 100644
index 00000000..daa4c52c
--- /dev/null
+++ b/febft-state-transfer/src/metrics/mod.rs
@@ -0,0 +1,13 @@
+use atlas_metrics::{MetricLevel, MetricRegistry};
+use atlas_metrics::metrics::MetricKind;
+
+/// State transfer will take the
+/// 6XX metric ID range
+pub const STATE_TRANSFER_STATE_INSTALL_CLONE_TIME : &str = "LT_STATE_CLONE_TIME";
+pub const STATE_TRANSFER_STATE_INSTALL_CLONE_TIME_ID : usize = 600;
+
+pub fn metrics() -> Vec<MetricRegistry> {
+    vec![
+        (STATE_TRANSFER_STATE_INSTALL_CLONE_TIME_ID, STATE_TRANSFER_STATE_INSTALL_CLONE_TIME.to_string(), MetricKind::Duration, MetricLevel::Info).into(),
+    ]
+}
\ No newline at end of file

From 780040ae7ceb03583712caeaa59397a1ec7422a3 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Fri, 23 Jun 2023 16:05:43 +0100
Subject: [PATCH 11/80] Done more work on persistent log handling.

---
 .../src/bft/consensus/decision/mod.rs              |  8 ++++----
 .../src/bft/msg_log/decided_log/mod.rs             | 14 +++++++-------
 febft-state-transfer/src/lib.rs                    |  4 ++--
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 7da99d39..10f6e088 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -12,7 +12,7 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::Node;
 use atlas_core::messages::ClientRqInfo;
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, WriteMode};
+use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, OperationMode};
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
@@ -351,7 +351,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
                                                                           stored_msg.header().digest().clone(),
                                                                           digests)?;
 
-                self.persistent_log.write_message(WriteMode::NonBlockingSync(None), stored_msg.clone())?;
+                self.persistent_log.write_message(OperationMode::NonBlockingSync(None), stored_msg.clone())?;
 
                 let mut result = DecisionStatus::Deciding;
 
@@ -454,7 +454,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
 
                 self.message_log.process_message(stored_msg.clone())?;
 
-                self.persistent_log.write_message(WriteMode::NonBlockingSync(None), stored_msg.clone())?;
+                self.persistent_log.write_message(OperationMode::NonBlockingSync(None), stored_msg.clone())?;
 
                 let mut result = DecisionStatus::Deciding;
 
@@ -529,7 +529,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
 
                 self.message_log.process_message(stored_msg.clone())?;
 
-                self.persistent_log.write_message(WriteMode::NonBlockingSync(None), stored_msg.clone())?;
+                self.persistent_log.write_message(OperationMode::NonBlockingSync(None), stored_msg.clone())?;
 
                 return if received == view.params().quorum() {
                     info!("{:?} // Completed commit phase with all commits Seq {:?} with commit from {:?}", node.id(), self.sequence_number(),
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index 0a8ae598..811edb02 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -10,7 +10,7 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::StoredMessage;
 use atlas_communication::Node;
 use atlas_core::ordering_protocol::{DecisionInformation, ExecutionResult, ProtocolConsensusDecision};
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, WriteMode};
+use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, OperationMode};
 use atlas_core::serialize::StateTransferMessage;
 use atlas_execution::app::{Request, UpdateBatch};
 use atlas_execution::serialize::ApplicationData;
@@ -58,7 +58,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// FIXME: The view initialization might have to be changed if we want to introduce reconfiguration
     pub fn read_current_state(&self, n: usize, f: usize) -> Result<Option<(ViewInfo, DecisionLog<D::Request>)>>
         where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
-        let option = self.persistent_log.read_state(WriteMode::BlockingSync)?;
+        let option = self.persistent_log.read_state(OperationMode::BlockingSync)?;
 
         if let Some((view, dec_log)) = option {
             Ok(Some((view, dec_log)))
@@ -87,7 +87,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     {
         if let Err(err) = self
             .persistent_log
-            .write_message(WriteMode::NonBlockingSync(None), consensus_msg)
+            .write_message(OperationMode::NonBlockingSync(None), consensus_msg)
         {
             error!("Failed to persist message {:?}", err);
         }
@@ -114,7 +114,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
         }
 
         if let Err(err) = self.persistent_log
-            .write_proof(WriteMode::NonBlockingSync(None), proof) {
+            .write_proof(OperationMode::NonBlockingSync(None), proof) {
             error!("Failed to persist proof {:?}", err);
         }
 
@@ -125,7 +125,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     pub fn clear_last_occurrence(&mut self, seq: SeqNo)
         where
             PL: OrderingProtocolLog<PBFTConsensus<D>> {
-        if let Err(err) = self.persistent_log.write_invalidate(WriteMode::NonBlockingSync(None), seq) {
+        if let Err(err) = self.persistent_log.write_invalidate(OperationMode::NonBlockingSync(None), seq) {
             error!("Failed to invalidate last occurrence {:?}", err);
         }
     }
@@ -140,7 +140,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
         let last_seq = self.dec_log.last_execution().unwrap_or(SeqNo::ZERO);
 
         if let Err(err) = self.persistent_log
-            .write_install_state(WriteMode::NonBlockingSync(None), view, dec_log) {
+            .write_install_state(OperationMode::NonBlockingSync(None), view, dec_log) {
             error!("Failed to persist message {:?}", err);
         }
     }
@@ -169,7 +169,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// Basically persists the metadata for a given consensus num
     pub fn all_batches_received(&mut self, metadata: ProofMetadata) where
         PL: OrderingProtocolLog<PBFTConsensus<D>> {
-        self.persistent_log.write_proof_metadata(WriteMode::NonBlockingSync(None),
+        self.persistent_log.write_proof_metadata(OperationMode::NonBlockingSync(None),
                                                  metadata).unwrap();
     }
 
diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 5ceccf1f..451dc42a 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -24,7 +24,7 @@ use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{StateTransfer, SystemMessage};
 use atlas_core::ordering_protocol::{ExecutionResult, OrderingProtocol, SerProof, View};
-use atlas_core::persistent_log::{MonolithicStateLog, PersistableStateTransferProtocol, WriteMode};
+use atlas_core::persistent_log::{MonolithicStateLog, PersistableStateTransferProtocol, OperationMode};
 use atlas_core::serialize::{LogTransferMessage, NetworkView, OrderingProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage, StateTransferMessage};
 use atlas_core::state_transfer::{Checkpoint, CstM, StateTransferProtocol, STResult, STTimeoutResult};
 use atlas_core::state_transfer::monolithic_state::MonolithicStateTransfer;
@@ -848,7 +848,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
                 self.current_checkpoint_state = checkpoint_state;
 
-                self.persistent_log.write_checkpoint(WriteMode::NonBlockingSync(None), checkpoint)?;
+                self.persistent_log.write_checkpoint(OperationMode::NonBlockingSync(None), checkpoint)?;
 
                 Ok(())
             }

From b0001c5bee4c9769262c96b3e6d8a849bdfd3807 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Tue, 27 Jun 2023 15:51:53 +0100
Subject: [PATCH 12/80] Fixed startup issue.

---
 .../src/bft/consensus/decision/mod.rs         |  8 +--
 .../src/bft/msg_log/decided_log/mod.rs        | 14 ++--
 febft-state-transfer/Cargo.toml               |  2 +-
 febft-state-transfer/src/lib.rs               | 69 +++++++++++--------
 .../src/message/serialize/capnp/mod.rs        | 13 ++--
 febft-state-transfer/src/metrics/mod.rs       | 13 ++++
 6 files changed, 72 insertions(+), 47 deletions(-)
 create mode 100644 febft-state-transfer/src/metrics/mod.rs

diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 7da99d39..10f6e088 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -12,7 +12,7 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::Node;
 use atlas_core::messages::ClientRqInfo;
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, WriteMode};
+use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, OperationMode};
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
@@ -351,7 +351,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
                                                                           stored_msg.header().digest().clone(),
                                                                           digests)?;
 
-                self.persistent_log.write_message(WriteMode::NonBlockingSync(None), stored_msg.clone())?;
+                self.persistent_log.write_message(OperationMode::NonBlockingSync(None), stored_msg.clone())?;
 
                 let mut result = DecisionStatus::Deciding;
 
@@ -454,7 +454,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
 
                 self.message_log.process_message(stored_msg.clone())?;
 
-                self.persistent_log.write_message(WriteMode::NonBlockingSync(None), stored_msg.clone())?;
+                self.persistent_log.write_message(OperationMode::NonBlockingSync(None), stored_msg.clone())?;
 
                 let mut result = DecisionStatus::Deciding;
 
@@ -529,7 +529,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
 
                 self.message_log.process_message(stored_msg.clone())?;
 
-                self.persistent_log.write_message(WriteMode::NonBlockingSync(None), stored_msg.clone())?;
+                self.persistent_log.write_message(OperationMode::NonBlockingSync(None), stored_msg.clone())?;
 
                 return if received == view.params().quorum() {
                     info!("{:?} // Completed commit phase with all commits Seq {:?} with commit from {:?}", node.id(), self.sequence_number(),
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index 0a8ae598..811edb02 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -10,7 +10,7 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::StoredMessage;
 use atlas_communication::Node;
 use atlas_core::ordering_protocol::{DecisionInformation, ExecutionResult, ProtocolConsensusDecision};
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, WriteMode};
+use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, OperationMode};
 use atlas_core::serialize::StateTransferMessage;
 use atlas_execution::app::{Request, UpdateBatch};
 use atlas_execution::serialize::ApplicationData;
@@ -58,7 +58,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// FIXME: The view initialization might have to be changed if we want to introduce reconfiguration
     pub fn read_current_state(&self, n: usize, f: usize) -> Result<Option<(ViewInfo, DecisionLog<D::Request>)>>
         where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
-        let option = self.persistent_log.read_state(WriteMode::BlockingSync)?;
+        let option = self.persistent_log.read_state(OperationMode::BlockingSync)?;
 
         if let Some((view, dec_log)) = option {
             Ok(Some((view, dec_log)))
@@ -87,7 +87,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     {
         if let Err(err) = self
             .persistent_log
-            .write_message(WriteMode::NonBlockingSync(None), consensus_msg)
+            .write_message(OperationMode::NonBlockingSync(None), consensus_msg)
         {
             error!("Failed to persist message {:?}", err);
         }
@@ -114,7 +114,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
         }
 
         if let Err(err) = self.persistent_log
-            .write_proof(WriteMode::NonBlockingSync(None), proof) {
+            .write_proof(OperationMode::NonBlockingSync(None), proof) {
             error!("Failed to persist proof {:?}", err);
         }
 
@@ -125,7 +125,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     pub fn clear_last_occurrence(&mut self, seq: SeqNo)
         where
             PL: OrderingProtocolLog<PBFTConsensus<D>> {
-        if let Err(err) = self.persistent_log.write_invalidate(WriteMode::NonBlockingSync(None), seq) {
+        if let Err(err) = self.persistent_log.write_invalidate(OperationMode::NonBlockingSync(None), seq) {
             error!("Failed to invalidate last occurrence {:?}", err);
         }
     }
@@ -140,7 +140,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
         let last_seq = self.dec_log.last_execution().unwrap_or(SeqNo::ZERO);
 
         if let Err(err) = self.persistent_log
-            .write_install_state(WriteMode::NonBlockingSync(None), view, dec_log) {
+            .write_install_state(OperationMode::NonBlockingSync(None), view, dec_log) {
             error!("Failed to persist message {:?}", err);
         }
     }
@@ -169,7 +169,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// Basically persists the metadata for a given consensus num
     pub fn all_batches_received(&mut self, metadata: ProofMetadata) where
         PL: OrderingProtocolLog<PBFTConsensus<D>> {
-        self.persistent_log.write_proof_metadata(WriteMode::NonBlockingSync(None),
+        self.persistent_log.write_proof_metadata(OperationMode::NonBlockingSync(None),
                                                  metadata).unwrap();
     }
 
diff --git a/febft-state-transfer/Cargo.toml b/febft-state-transfer/Cargo.toml
index c354226d..aca2a5cf 100644
--- a/febft-state-transfer/Cargo.toml
+++ b/febft-state-transfer/Cargo.toml
@@ -23,4 +23,4 @@ atlas-execution = { path = "../../Atlas/atlas-execution" }
 atlas-common = { path = "../../Atlas/atlas-common" }
 atlas-communication = { path = "../../Atlas/atlas-communication" }
 atlas-core = { path = "../../Atlas/atlas-core" }
-atlas-persistent-log = { path = "../../Atlas/atlas-persistent-log" }
\ No newline at end of file
+atlas-metrics = { path = "../../Atlas/atlas-metrics" }
\ No newline at end of file
diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index b657bc20..a42dae1a 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -3,7 +3,7 @@
 use std::cmp::Ordering;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use log::{debug, error, info};
 #[cfg(feature = "serialize_serde")]
@@ -24,19 +24,22 @@ use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{StateTransfer, SystemMessage};
 use atlas_core::ordering_protocol::{ExecutionResult, OrderingProtocol, SerProof, View};
-use atlas_core::persistent_log::{MonolithicStateLog, PersistableStateTransferProtocol, WriteMode};
+use atlas_core::persistent_log::{MonolithicStateLog, PersistableStateTransferProtocol, OperationMode};
 use atlas_core::serialize::{LogTransferMessage, NetworkView, OrderingProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage, StateTransferMessage};
 use atlas_core::state_transfer::{Checkpoint, CstM, StateTransferProtocol, STResult, STTimeoutResult};
 use atlas_core::state_transfer::monolithic_state::MonolithicStateTransfer;
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, Timeouts};
 use atlas_execution::state::monolithic_state::{InstallStateMessage, MonolithicState};
+use atlas_metrics::metrics::metric_duration;
 
 use crate::config::StateTransferConfig;
 use crate::message::{CstMessage, CstMessageKind};
 use crate::message::serialize::CSTMsg;
+use crate::metrics::STATE_TRANSFER_STATE_INSTALL_CLONE_TIME_ID;
 
 pub mod message;
 pub mod config;
+pub mod metrics;
 
 /// The state of the checkpoint
 pub enum CheckpointState<D> {
@@ -278,7 +281,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
             CstStatus::Nil => (),
 // should not happen...
             _ => {
-                return Err("Invalid state reached!").wrapped(ErrorKind::CoreServer);
+                return Err(format!("Invalid state reached while state transfer processing message! {:?}", status)).wrapped(ErrorKind::CoreServer);
             }
         }
 
@@ -323,8 +326,12 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         match status {
             CstStatus::Running => (),
             CstStatus::State(state) => {
+                let start = Instant::now();
+
                 self.install_channel.send(InstallStateMessage::new(state.checkpoint.state().clone())).unwrap();
 
+                metric_duration(STATE_TRANSFER_STATE_INSTALL_CLONE_TIME_ID, start.elapsed());
+
                 return Ok(STResult::StateTransferFinished(state.checkpoint.sequence_number()));
             }
             CstStatus::SeqNo(seq) => {
@@ -344,10 +351,10 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
                 self.request_latest_state(view);
             }
             CstStatus::Nil => {
-// No actions are required for the CST
-// This can happen for example when we already received the a quorum of sequence number replies
-// And therefore we are already in the Init phase and we are still receiving replies
-// And have not yet processed the
+                // No actions are required for the CST
+                // This can happen for example when we already received the a quorum of sequence number replies
+                // And therefore we are already in the Init phase and we are still receiving replies
+                // And have not yet processed the
             }
         }
 
@@ -510,7 +517,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>{
+              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>> {
         let waiting = std::mem::replace(&mut self.phase, ProtoPhase::Init);
 
         if let ProtoPhase::WaitingCheckpoint(reqs) = waiting {
@@ -649,27 +656,27 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
                 match message.kind() {
                     CstMessageKind::ReplyStateCid(state_cid) => {
-
                         if let Some((cid, digest)) = state_cid {
-
-                        } else {
-
-                        }
-                        todo!();
-
-                        /*debug!("{:?} // Received CID vote {:?} from {:?}", self.node.id(), seq, header.from());
-
-                        match seq.cmp(&self.largest_cid) {
-                            Ordering::Greater => {
-                                self.largest_cid = seq;
-                                self.latest_cid_count = 1;
-                            }
-                            Ordering::Equal => {
-                                self.latest_cid_count += 1;
+                            let received_state_cid = self.received_state_ids.entry(digest.clone()).or_insert_with(|| {
+                                ReceivedStateCid {
+                                    cid: *cid,
+                                    count: 0,
+                                }
+                            });
+
+                            if *cid > received_state_cid.cid {
+                                info!("{:?} // Received newer state for old cid {:?} vs new cid {:?} with digest {:?}.",
+                                    self.node.id(), received_state_cid.cid, *cid, digest);
+
+                                received_state_cid.cid = *cid;
+                                received_state_cid.count = 1;
+                            } else if *cid == received_state_cid.cid {
+                                info!("{:?} // Received matching state for cid {:?} with digest {:?}. Count {}",
+                                self.node.id(), received_state_cid.cid, digest, received_state_cid.count + 1);
+
+                                received_state_cid.count += 1;
                             }
-                            Ordering::Less => (),
                         }
-                         */
                     }
                     CstMessageKind::RequestStateCid => {
                         self.process_request_seq(header, message);
@@ -701,7 +708,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                     // reset timeout, since req was successful
                     self.curr_timeout = self.base_timeout;
 
-                    info!("{:?} // Identified the latest consensus seq no as {:?} with {} votes.",
+                    info!("{:?} // Identified the latest state seq no as {:?} with {} votes.",
                             self.node.id(),
                             self.largest_cid, self.latest_cid_count);
 
@@ -841,7 +848,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
                 self.current_checkpoint_state = checkpoint_state;
 
-                self.persistent_log.write_checkpoint(WriteMode::NonBlockingSync(None), checkpoint)?;
+                self.persistent_log.write_checkpoint(OperationMode::NonBlockingSync(None), checkpoint)?;
 
                 Ok(())
             }
@@ -930,9 +937,11 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         self.largest_cid = SeqNo::ZERO;
         self.latest_cid_count = 0;
 
+        self.next_seq();
+
         let cst_seq = self.curr_seq();
 
-        info!("{:?} // Requesting latest consensus seq no with seq {:?}", self.node.id(), cst_seq);
+        info!("{:?} // Requesting latest state seq no with seq {:?}", self.node.id(), cst_seq);
 
         self.timeouts.timeout_cst_request(self.curr_timeout,
                                           view.quorum() as u32,
@@ -962,6 +971,8 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         // reset hashmap of received states
         self.received_states.clear();
 
+        self.next_seq();
+
         let cst_seq = self.curr_seq();
 
         info!("{:?} // Requesting latest state with cst msg seq {:?}", self.node.id(), cst_seq);
diff --git a/febft-state-transfer/src/message/serialize/capnp/mod.rs b/febft-state-transfer/src/message/serialize/capnp/mod.rs
index 366799e9..b2ba29fa 100644
--- a/febft-state-transfer/src/message/serialize/capnp/mod.rs
+++ b/febft-state-transfer/src/message/serialize/capnp/mod.rs
@@ -1,16 +1,17 @@
 use atlas_execution::serialize::ApplicationData;
 use atlas_common::error::*;
 use atlas_core::state_transfer::StatefulOrderProtocol;
+use atlas_execution::state::divisible_state::DivisibleState;
 use crate::message::CstMessage;
 
-fn serialize_state_transfer<D, SOP, NT>(mut state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Builder,
-                                    msg: &CstMessage<D::State, SOP::DecLog, SOP::ViewInfo>) -> Result<()>
-    where D: ApplicationData, SOP: StatefulOrderProtocol<D, NT> {
+fn serialize_state_transfer<S>(mut state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Builder,
+                               msg: &CstMessage<S>) -> Result<()>
+    where S: DivisibleState {
     Ok(())
 }
 
-fn deserialize_state_transfer<D, SOP, NT>(state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Reader)
-                                      -> Result<CstMessage<D::State, SOP::DecLog, SOP::ViewInfo>>
-    where D: ApplicationData, SOP: StatefulOrderProtocol<D, NT> {
+fn deserialize_state_transfer<S>(state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Reader)
+                                 -> Result<CstMessage<S>>
+    where S: DivisibleState {
     Err(Error::simple(ErrorKind::CommunicationSerialize))
 }
diff --git a/febft-state-transfer/src/metrics/mod.rs b/febft-state-transfer/src/metrics/mod.rs
new file mode 100644
index 00000000..daa4c52c
--- /dev/null
+++ b/febft-state-transfer/src/metrics/mod.rs
@@ -0,0 +1,13 @@
+use atlas_metrics::{MetricLevel, MetricRegistry};
+use atlas_metrics::metrics::MetricKind;
+
+/// State transfer will take the
+/// 6XX metric ID range
+pub const STATE_TRANSFER_STATE_INSTALL_CLONE_TIME : &str = "LT_STATE_CLONE_TIME";
+pub const STATE_TRANSFER_STATE_INSTALL_CLONE_TIME_ID : usize = 600;
+
+pub fn metrics() -> Vec<MetricRegistry> {
+    vec![
+        (STATE_TRANSFER_STATE_INSTALL_CLONE_TIME_ID, STATE_TRANSFER_STATE_INSTALL_CLONE_TIME.to_string(), MetricKind::Duration, MetricLevel::Info).into(),
+    ]
+}
\ No newline at end of file

From 44c9c190b05073168bf765da13d36ffc92a37618 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 29 Jun 2023 21:19:02 +0100
Subject: [PATCH 13/80] Fix bug that made proposer discard some requests when
 he really shouldn't. Fixed some more random things.

---
 febft-pbft-consensus/src/bft/consensus/mod.rs |  4 ++
 febft-pbft-consensus/src/bft/proposer/mod.rs  | 62 +++++++------------
 febft-pbft-consensus/src/bft/sync/mod.rs      |  2 +-
 3 files changed, 29 insertions(+), 39 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 479874dc..3bf41747 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -926,6 +926,8 @@ impl ProposerConsensusGuard {
     /// Lock the consensus, making it impossible for the proposer to propose any requests
     pub fn lock_consensus(&self) {
         self.can_propose.store(false, Ordering::Relaxed);
+
+        debug!("Locked consensus");
     }
 
     /// Unlock the consensus instance
@@ -933,6 +935,8 @@ impl ProposerConsensusGuard {
         self.can_propose.store(true, Ordering::Relaxed);
 
         self.event_waker.notify(usize::MAX);
+
+        debug!("Unlocking consensus")
     }
 
     /// Get the next sequence number to propose to
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index bbc1d001..e56f726f 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -140,16 +140,6 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
                         info!("{:?} // Resuming proposer as we are now able to propose again.", self.node_ref.id());
                     }
 
-                    //TODO: Maybe not use this as it can spam the lock on synchronizer?
-                    let info = self.synchronizer.view();
-
-                    let is_leader = info.leader_set().contains(&self.node_ref.id());
-
-                    let leader_set_size = info.leader_set().len();
-
-                    let our_slice = info.hash_space_division()
-                        .get(&self.node_ref.id()).cloned().clone();
-
                     //We do this as we don't want to get stuck waiting for requests that might never arrive
                     //Or even just waiting for any type of request. We want to minimize the amount of time the
                     //Consensus is waiting for new requests
@@ -157,38 +147,31 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
                     //We don't need to do this for non leader replicas, as that would cause unnecessary strain as the
                     //Thread is in an infinite loop
                     // Receive the requests from the clients and process them
-                    let opt_msgs: Option<PreProcessorOutputMessage<D::Request>> = if is_leader {
-                        match self.batch_reception.try_recv() {
-                            Ok(res) => { Some(res) }
-                            Err(err) => {
-                                match err {
-                                    TryRecvError::ChannelDc => {
-                                        error!("{:?} // Failed to receive requests from pre processing module because {:?}", self.node_ref.id(), err);
-                                        break;
-                                    }
-                                    _ => {
-                                        None
-                                    }
+                    let opt_msgs: Option<PreProcessorOutputMessage<D::Request>> = match self.batch_reception.try_recv() {
+                        Ok(res) => { Some(res) }
+                        Err(err) => {
+                            match err {
+                                TryRecvError::ChannelDc => {
+                                    error!("{:?} // Failed to receive requests from pre processing module because {:?}", self.node_ref.id(), err);
+                                    break;
                                 }
-                            }
-                        }
-                    } else {
-                        match self.batch_reception.recv_timeout(Duration::from_micros(self.global_batch_time_limit as u64)) {
-                            Ok(res) => { Some(res) }
-                            Err(err) => {
-                                match err {
-                                    TryRecvError::Timeout | TryRecvError::ChannelEmpty => {
-                                        None
-                                    }
-                                    TryRecvError::ChannelDc => {
-                                        error!("{:?} // Failed to receive requests from pre processing module because {:?}", self.node_ref.id(), err);
-                                        break;
-                                    }
+                                _ => {
+                                    None
                                 }
                             }
                         }
                     };
 
+                    //TODO: Maybe not use this as it can spam the lock on synchronizer?
+                    let info = self.synchronizer.view();
+
+                    let is_leader = info.leader_set().contains(&self.node_ref.id());
+
+                    let leader_set_size = info.leader_set().len();
+
+                    let our_slice = info.hash_space_division()
+                        .get(&self.node_ref.id()).cloned().clone();
+
                     let discovered_requests;
 
                     if let Some(messages) = opt_msgs {
@@ -335,8 +318,8 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
     /// attempt to propose the ordered requests that we have collected
     /// Returns true if a batch was proposed
     fn propose_ordered<ST, LP>(&self,
-                           is_leader: bool,
-                           propose: &mut ProposeBuilder<D>, ) -> bool
+                               is_leader: bool,
+                               propose: &mut ProposeBuilder<D>, ) -> bool
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
               NT: Node<PBFT<D, ST, LP>> {
@@ -413,6 +396,9 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
             currently_accumulated.retain(|msg| {
                 if self.check_if_has_been_proposed(msg, &mut view_change_msg) {
                     // if it has been proposed, then we do not want to retain it
+
+                    let info1 = ClientRqInfo::from(msg);
+                    debug!("{:?} // Request {:?} has already been proposed, not retaining it", info1, self.node_ref.id());
                     return false;
                 }
                 true
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index a413e29f..f9517c61 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -829,7 +829,7 @@ impl<D> Synchronizer<D>
                             let (header, message) = {
                                 let mut buf = Vec::new();
 
-                                info!("{:?} // Forged pre-prepare: {} {:?}", node.id(), p.len(), p);
+                                info!("{:?} // Forged pre-prepare: {}", node.id(), p.len());
 
                                 let forged_pre_prepare = consensus.forge_propose(p, self);
 

From 7afd75b4283b23b4560a76bdde2097ce1c78c4d4 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 6 Jul 2023 18:16:35 +0100
Subject: [PATCH 14/80] Started testing the networking part of the
 reconfiguration logic.

---
 .../src/bft/consensus/accessory/replica/mod.rs                | 4 ++--
 febft-pbft-consensus/src/bft/mod.rs                           | 3 +--
 febft-pbft-consensus/src/bft/sync/mod.rs                      | 4 ++--
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index fbbea87d..10de3b28 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -45,7 +45,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
         let view_seq = view.sequence_number();
         let current_digest = deciding_log.current_digest().unwrap();
 
-        let sign_detached = node.pk_crypto().sign_detached();
+        let key_pair = node.pk_crypto().get_key_pair().clone();
         let n = view.params().n();
 
         let seq = deciding_log.sequence_number();
@@ -84,7 +84,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
                     // PRE-PREPARE message
                     0,
                     Some(digest),
-                    Some(sign_detached.key_pair()),
+                    Some(&*key_pair),
                 ).into_inner();
 
                 // store serialized header + message
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 9f776096..b7860ef0 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -16,7 +16,6 @@ use log4rs::{
     encode::pattern::PatternEncoder,
 };
 
-
 use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::ordering::{Orderable, SeqNo};
@@ -28,11 +27,11 @@ use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::messages::Protocol;
 use atlas_core::ordering_protocol::{OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, ProtocolConsensusDecision, ProtocolMessage, SerProof, SerProofMetadata, View};
+use atlas_core::ordering_protocol::stateful_order_protocol::{DecLog, StatefulOrderProtocol};
 use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol, StatefulOrderingProtocolLog};
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
 use atlas_core::serialize::{LogTransferMessage, NetworkView, ServiceMsg, StateTransferMessage};
 use atlas_core::state_transfer::{Checkpoint};
-use atlas_core::state_transfer::log_transfer::{DecLog, StatefulOrderProtocol};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_metrics::metrics::metric_duration;
 use crate::bft::config::PBFTConfig;
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index f9517c61..5733a6ad 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -822,7 +822,7 @@ impl<D> Synchronizer<D>
                             }
 
                             let p = rq_pre_processor.collect_all_pending_rqs();
-                            let node_sign = node.pk_crypto().sign_detached();
+                            let node_sign = node.pk_crypto().get_key_pair().clone();
 
                             //We create the pre-prepare here as we are the new leader,
                             //And we sign it right now
@@ -852,7 +852,7 @@ impl<D> Synchronizer<D>
                                     buf,
                                     prng_state.next_state(),
                                     Some(digest),
-                                    Some(node_sign.key_pair()),
+                                    Some(&*node_sign),
                                 ).into_inner();
 
                                 if let PBFTMessage::Consensus(consensus) = forged_pre_prepare.into_system().into_protocol_message() {

From 9a0a72465fd8266392c57789c4df9393a4491912 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 10 Jul 2023 15:09:58 +0100
Subject: [PATCH 15/80] Separate the reconfiguration from the protocol so we
 don't need any extra generics in our protocols. Abstract reconfiguration from
 communication so it doesn't have to be handled there and can now be treated
 separately.

---
 .../src/bft/consensus/accessory/replica/mod.rs            | 6 +++---
 febft-pbft-consensus/src/bft/proposer/mod.rs              | 2 +-
 febft-pbft-consensus/src/bft/sync/mod.rs                  | 4 ++--
 febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs     | 6 +++---
 febft-state-transfer/src/lib.rs                           | 8 ++++----
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index 10de3b28..89ba73e4 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -53,7 +53,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
         let speculative_commits = Arc::clone(&self.speculative_commits);
 
         threadpool::execute(move || {
-            let message = NetworkMessageKind::from(
+            let message = NetworkMessageKind::from_system(
                 SystemMessage::from_protocol_message(
                     PBFTMessage::Consensus(ConsensusMessage::new(
                         seq,
@@ -115,7 +115,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
 
         let targets = NodeId::targets(0..view.params().n());
 
-        node.broadcast_signed(NetworkMessageKind::from(SystemMessage::from_protocol_message(message)), targets);
+        node.broadcast_signed(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), targets);
     }
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
@@ -151,7 +151,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
 
             let targets = NodeId::targets(0..view.params().n());
 
-            node.broadcast_signed(NetworkMessageKind::from(SystemMessage::from_protocol_message(message)), targets);
+            node.broadcast_signed(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), targets);
         }
 
         debug!("{:?} // Broadcasted commit consensus message {:?}",
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index e56f726f..5807912a 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -423,7 +423,7 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
 
         let targets = view.quorum_members().iter().copied();
 
-        self.node_ref.broadcast_signed(NetworkMessageKind::from(SystemMessage::from_protocol_message(message)), targets);
+        self.node_ref.broadcast_signed(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), targets);
 
         metric_increment(PROPOSER_BATCHES_MADE_ID, Some(1));
     }
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 5733a6ad..7c1e9b70 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -833,7 +833,7 @@ impl<D> Synchronizer<D>
 
                                 let forged_pre_prepare = consensus.forge_propose(p, self);
 
-                                let forged_pre_prepare = NetworkMessageKind::from(forged_pre_prepare);
+                                let forged_pre_prepare = NetworkMessageKind::from_system(forged_pre_prepare);
 
                                 let digest = serialize::serialize_digest::<Vec<u8>, PBFT<D, ST, LP>>(
                                     &forged_pre_prepare,
@@ -880,7 +880,7 @@ impl<D> Synchronizer<D>
                             let targets = NodeId::targets(0..current_view.params().n())
                                 .filter(move |&id| id != node_id);
 
-                            node.broadcast(NetworkMessageKind::from(SystemMessage::from_protocol_message(message)), targets);
+                            node.broadcast(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), targets);
 
                             let state = FinalizeState {
                                 curr_cid,
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 8642bfee..fd369c1b 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -109,7 +109,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
             ViewChangeMessageKind::StopData(collect),
         ));
 
-        node.send_signed(NetworkMessageKind::from(SystemMessage::from_protocol_message(message)), current_leader, true);
+        node.send_signed(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), current_leader, true);
     }
 
     /// Start a new view change
@@ -146,7 +146,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
 
         let targets = NodeId::targets(0..current_view.params().n());
 
-        node.broadcast(NetworkMessageKind::from(SystemMessage::from_protocol_message(message)), targets);
+        node.broadcast(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), targets);
     }
 
     /// Watch a vector of requests received
@@ -346,7 +346,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
             NT: Node<PBFT<D, ST, LP>> {
         let message = SystemMessage::ForwardedRequestMessage(ForwardedRequestsMessage::new(timed_out));
         let targets = NodeId::targets(0..base_sync.view().params().n());
-        node.broadcast(NetworkMessageKind::from(message), targets);
+        node.broadcast(NetworkMessageKind::from_system(message), targets);
     }
 
     /// Obtain the requests that we know have timed out so we can send out a stop message
diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index a42dae1a..3295355e 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -502,7 +502,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
         let reply = CstMessage::new(message.sequence_number(), kind);
 
-        let network_msg = NetworkMessageKind::from(SystemMessage::from_state_transfer_message(reply));
+        let network_msg = NetworkMessageKind::from_system(SystemMessage::from_state_transfer_message(reply));
 
         debug!("{:?} // Replying to {:?} seq {:?} with seq no {:?}", self.node.id(),
             header.from(), message.sequence_number(), seq);
@@ -592,7 +592,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
             }),
         );
 
-        let network_msg = NetworkMessageKind::from(SystemMessage::from_state_transfer_message(reply));
+        let network_msg = NetworkMessageKind::from_system(SystemMessage::from_state_transfer_message(reply));
 
         self.node.send(network_msg, header.from(), true).unwrap();
     }
@@ -956,7 +956,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
         let targets = NodeId::targets(0..view.n());
 
-        self.node.broadcast(NetworkMessageKind::from(SystemMessage::from_state_transfer_message(message)), targets);
+        self.node.broadcast(NetworkMessageKind::from_system(SystemMessage::from_state_transfer_message(message)), targets);
     }
 
     /// Used by a recovering node to retrieve the latest state.
@@ -988,7 +988,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         let message = CstMessage::new(cst_seq, CstMessageKind::RequestState);
         let targets = NodeId::targets(0..view.n()).filter(|id| *id != self.node.id());
 
-        self.node.broadcast(NetworkMessageKind::from(SystemMessage::from_state_transfer_message(message)), targets);
+        self.node.broadcast(NetworkMessageKind::from_system(SystemMessage::from_state_transfer_message(message)), targets);
     }
 }
 

From 36852e50c5f1b79382d1e61f8ad1face90d71d7f Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 13 Jul 2023 01:52:34 +0100
Subject: [PATCH 16/80] Moved more things to the new abstractions of the
 networking

---
 febft-state-transfer/src/lib.rs | 36 ++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 3295355e..01e58734 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -17,8 +17,8 @@ use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_communication::Node;
 use atlas_communication::message::{Header, NetworkMessageKind, StoredMessage};
+use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_execution::app::{Application, Reply, Request};
 use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
@@ -238,7 +238,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>,
               V: NetworkView {
         self.request_latest_consensus_seq_no::<D, OP, LP, V>(view);
 
@@ -250,7 +250,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>,
               V: NetworkView {
         let (header, message) = message.into_inner();
 
@@ -295,7 +295,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>,
               V: NetworkView {
         let (header, message) = message.into_inner();
 
@@ -365,7 +365,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>, V: NetworkView {
+              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>, V: NetworkView {
         let earlier = std::mem::replace(&mut self.current_checkpoint_state, CheckpointState::None);
 
         self.current_checkpoint_state = match earlier {
@@ -392,7 +392,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>,
               V: NetworkView {
         for cst_seq in timeout {
             if let TimeoutKind::Cst(cst_seq) = cst_seq.timeout_kind() {
@@ -426,7 +426,7 @@ impl<S, NT, PL> MonolithicStateTransfer<S, NT, PL> for CollabStateTransfer<S, NT
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Self::Serialization, LP>>,
+              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>,
               V: NetworkView {
         self.finalize_checkpoint(state)?;
 
@@ -484,7 +484,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>
+              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>
     {
         let seq = match &self.current_checkpoint_state {
             CheckpointState::PartialWithEarlier { seq, earlier, } => {
@@ -502,7 +502,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
         let reply = CstMessage::new(message.sequence_number(), kind);
 
-        let network_msg = NetworkMessageKind::from_system(SystemMessage::from_state_transfer_message(reply));
+        let network_msg = SystemMessage::from_state_transfer_message(reply);
 
         debug!("{:?} // Replying to {:?} seq {:?} with seq no {:?}", self.node.id(),
             header.from(), message.sequence_number(), seq);
@@ -517,7 +517,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>> {
+              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>> {
         let waiting = std::mem::replace(&mut self.phase, ProtoPhase::Init);
 
         if let ProtoPhase::WaitingCheckpoint(reqs) = waiting {
@@ -554,7 +554,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     ) where D: ApplicationData + 'static,
             OP: OrderingProtocolMessage + 'static,
             LP: LogTransferMessage + 'static,
-            NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>
+            NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>
     {
         match &mut self.phase {
             ProtoPhase::Init => {}
@@ -592,7 +592,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
             }),
         );
 
-        let network_msg = NetworkMessageKind::from_system(SystemMessage::from_state_transfer_message(reply));
+        let network_msg = SystemMessage::from_state_transfer_message(reply);
 
         self.node.send(network_msg, header.from(), true).unwrap();
     }
@@ -606,7 +606,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
+              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
               V: NetworkView
     {
         match self.phase {
@@ -872,7 +872,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         where D: ApplicationData + 'static,
               OP: OrderingProtocolMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>, V: NetworkView {
+              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>, V: NetworkView {
         let status = self.timed_out(seq);
 
         match status {
@@ -929,7 +929,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     ) where D: ApplicationData + 'static,
             OP: OrderingProtocolMessage + 'static,
             LP: LogTransferMessage + 'static,
-            NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
+            NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
             V: NetworkView
     {
 
@@ -956,7 +956,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
         let targets = NodeId::targets(0..view.n());
 
-        self.node.broadcast(NetworkMessageKind::from_system(SystemMessage::from_state_transfer_message(message)), targets);
+        self.node.broadcast(SystemMessage::from_state_transfer_message(message), targets);
     }
 
     /// Used by a recovering node to retrieve the latest state.
@@ -965,7 +965,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     ) where D: ApplicationData + 'static,
             OP: OrderingProtocolMessage + 'static,
             LP: LogTransferMessage + 'static,
-            NT: Node<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
+            NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
             V: NetworkView
     {
         // reset hashmap of received states
@@ -988,7 +988,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         let message = CstMessage::new(cst_seq, CstMessageKind::RequestState);
         let targets = NodeId::targets(0..view.n()).filter(|id| *id != self.node.id());
 
-        self.node.broadcast(NetworkMessageKind::from_system(SystemMessage::from_state_transfer_message(message)), targets);
+        self.node.broadcast(SystemMessage::from_state_transfer_message(message), targets);
     }
 }
 

From 8a4d8196a568080cd97db7d9a644a18c07174c14 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 13 Jul 2023 19:37:37 +0100
Subject: [PATCH 17/80] Fix compile issues with the new network abstractions.

---
 febft-pbft-consensus/src/bft/config/mod.rs    | 13 ++--
 .../src/bft/consensus/accessory/mod.rs        | 33 +++++----
 .../bft/consensus/accessory/replica/mod.rs    | 72 +++++++++----------
 .../src/bft/consensus/decision/mod.rs         | 26 +++----
 febft-pbft-consensus/src/bft/consensus/mod.rs | 40 +++++------
 .../src/bft/message/serialize/mod.rs          | 17 ++---
 febft-pbft-consensus/src/bft/mod.rs           | 24 +++----
 .../src/bft/msg_log/decided_log/mod.rs        | 20 +++---
 .../src/bft/proposer/follower_proposer/mod.rs | 15 ++--
 febft-pbft-consensus/src/bft/proposer/mod.rs  | 50 +++++++------
 .../src/bft/sync/follower_sync/mod.rs         |  7 +-
 febft-pbft-consensus/src/bft/sync/mod.rs      | 60 +++++++---------
 .../src/bft/sync/replica_sync/mod.rs          | 23 +++---
 13 files changed, 183 insertions(+), 217 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/config/mod.rs b/febft-pbft-consensus/src/bft/config/mod.rs
index 9c4a8f01..8364abc5 100644
--- a/febft-pbft-consensus/src/bft/config/mod.rs
+++ b/febft-pbft-consensus/src/bft/config/mod.rs
@@ -1,17 +1,12 @@
 use std::marker::PhantomData;
-use std::sync::Arc;
 use std::time::Duration;
-use atlas_common::globals::ReadOnly;
+
 use atlas_common::node_id::NodeId;
-use atlas_communication::Node;
-use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::ApplicationData;
 use atlas_core::followers::FollowerHandle;
-use atlas_core::serialize::{OrderingProtocolMessage, StateTransferMessage, ServiceMsg};
-use atlas_core::state_transfer::Checkpoint;
-use atlas_core::timeouts::Timeouts;
+use atlas_core::serialize::{OrderingProtocolMessage, StateTransferMessage};
+use atlas_execution::serialize::ApplicationData;
+
 use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::observer::ObserverHandle;
 use crate::bft::sync::view::ViewInfo;
 
 pub struct PBFTConfig<D: ApplicationData, ST> {
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
index a0412e05..db7d6905 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
@@ -1,10 +1,9 @@
-use atlas_common::crypto::hash::Digest;
-use atlas_common::ordering::SeqNo;
-use atlas_communication::Node;
-use atlas_execution::serialize::ApplicationData;
+use std::sync::Arc;
+use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
+use atlas_execution::serialize::ApplicationData;
+
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
-use crate::bft::message::ConsensusMessage;
 use crate::bft::msg_log::deciding_log::DecidingLog;
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
 use crate::bft::PBFT;
@@ -25,40 +24,40 @@ pub trait AccessoryConsensus<D, ST, LP> where D: ApplicationData + 'static,
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: Node<PBFT<D, ST, LP>>;
+                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>>;
 
     /// Handle the prepare phase having been completed
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
-                                              node: &NT) where NT: Node<PBFT<D, ST, LP>>;
+                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static;
 
     /// Handle a prepare message processed during the preparing phase without having
     /// reached a quorum
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: Node<PBFT<D, ST, LP>>;
+                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>>;
 
     /// Handle a prepare message processed during the prepare phase when a quorum
     /// has been achieved
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
-                                   node: &NT) where NT: Node<PBFT<D, ST, LP>>;
+                                   node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>>;
 
     /// Handle a commit message processed during the preparing phase without having
     /// reached a quorum
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
-                                       node: &NT) where NT: Node<PBFT<D, ST, LP>>;
+                                       node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>>;
 
     /// Handle a commit message processed during the prepare phase when a quorum has been reached
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
-                                    node: &NT) where NT: Node<PBFT<D, ST, LP>>;
+                                    node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>>;
 }
 
 impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D, ST, LP>
@@ -68,7 +67,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: Node<PBFT<D, ST, LP>> {
+                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -80,7 +79,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
-                                              node: &NT) where NT: Node<PBFT<D, ST, LP>> {
+                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -92,7 +91,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: Node<PBFT<D, ST, LP>> {
+                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -104,7 +103,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
-                                   node: &NT) where NT: Node<PBFT<D, ST, LP>> {
+                                   node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -116,7 +115,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
-                                       node: &NT) where NT: Node<PBFT<D, ST, LP>> {
+                                       node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -128,7 +127,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
-                                    node: &NT) where NT: Node<PBFT<D, ST, LP>> {
+                                    node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index 89ba73e4..5488deb0 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -1,30 +1,32 @@
 use std::collections::BTreeMap;
 use std::ops::Deref;
 use std::sync::{Arc, Mutex};
+
 use chrono::Utc;
 use log::debug;
-use atlas_common::crypto::hash::Digest;
+
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::threadpool;
-use atlas_communication::message::{NetworkMessageKind, SerializedMessage, StoredMessage, StoredSerializedNetworkMessage, WireMessage};
-use atlas_communication::{Node, NodePK, serialize};
-use atlas_communication::serialize::Buf;
-use atlas_execution::serialize::ApplicationData;
+use atlas_communication::message::{NetworkMessageKind, SerializedMessage, StoredMessage, StoredSerializedProtocolMessage, WireMessage};
+use atlas_communication::protocol_node::ProtocolNetworkNode;
+use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::messages::SystemMessage;
-use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, ServiceMsg, StateTransferMessage};
+use atlas_execution::serialize::ApplicationData;
+
+use crate::bft::consensus::accessory::AccessoryConsensus;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::msg_log::deciding_log::DecidingLog;
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
-use crate::bft::PBFT;
+use crate::bft::{PBFT, SysMsg};
 use crate::bft::sync::view::ViewInfo;
-use crate::bft::consensus::accessory::AccessoryConsensus;
 
 pub struct ReplicaAccessory<D, ST, LP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static {
-    speculative_commits: Arc<Mutex<BTreeMap<NodeId, StoredSerializedNetworkMessage<PBFT<D, ST, LP>>>>>,
+    speculative_commits: Arc<Mutex<BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP>>>>>,
 }
 
 impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
@@ -34,41 +36,37 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: Node<PBFT<D, ST, LP>> {}
+                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {}
 
     fn handle_pre_prepare_phase_completed<NT>(&mut self,
                                               deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               _msg: StoredConsensusMessage<D::Request>,
-                                              node: &NT) where NT: Node<PBFT<D, ST, LP>> {
+                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static {
         let my_id = node.id();
         let view_seq = view.sequence_number();
         let current_digest = deciding_log.current_digest().unwrap();
 
-        let key_pair = node.pk_crypto().get_key_pair().clone();
+        let key_pair = node.network_info_provider().get_key_pair().clone();
         let n = view.params().n();
 
         let seq = deciding_log.sequence_number();
 
         let speculative_commits = Arc::clone(&self.speculative_commits);
 
-        threadpool::execute(move || {
-            let message = NetworkMessageKind::from_system(
-                SystemMessage::from_protocol_message(
-                    PBFTMessage::Consensus(ConsensusMessage::new(
-                        seq,
-                        view_seq,
-                        ConsensusMessageKind::Commit(current_digest.clone()),
-                    ))));
+        let node_clone = node.clone();
 
+        threadpool::execute(move || {
+            let message = SystemMessage::from_protocol_message(
+                PBFTMessage::Consensus(ConsensusMessage::new(
+                    seq,
+                    view_seq,
+                    ConsensusMessageKind::Commit(current_digest.clone()),
+                )));
 
-            // serialize raw msg
-            let mut buf = Vec::new();
-
-            let digest = serialize::serialize_digest::<Vec<u8>,
-                PBFT<D, ST, LP>>(&message, &mut buf).unwrap();
+            let (message, digest) = node_clone.serialize_digest_message(message).unwrap();
 
-            let buf = Buf::from(buf);
+            let (message, buf) = message.into_inner();
 
             for peer_id in NodeId::targets(0..n) {
                 let buf_clone = buf.clone();
@@ -115,16 +113,16 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
 
         let targets = NodeId::targets(0..view.params().n());
 
-        node.broadcast_signed(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), targets);
+        node.broadcast_signed(SystemMessage::from_protocol_message(message), targets);
     }
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST, LP>> {}
+                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {}
 
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
-                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST, LP>> {
+                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         let node_id = node.id();
 
         let seq = deciding_log.sequence_number();
@@ -133,8 +131,8 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
 
         if valid_spec_commits::<D, ST, LP>(&speculative_commits, node_id, seq, view) {
             for (_, msg) in speculative_commits.iter() {
-                debug!("{:?} // Broadcasting speculative commit message {:?} (total of {} messages) to {} targets",
-                     node_id, msg.message().original(), speculative_commits.len(), view.params().n());
+                debug!("{:?} // Broadcasting speculative commit message (total of {} messages) to {} targets",
+                     node_id, speculative_commits.len(), view.params().n());
                 break;
             }
 
@@ -151,7 +149,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
 
             let targets = NodeId::targets(0..view.params().n());
 
-            node.broadcast_signed(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), targets);
+            node.broadcast_signed(SystemMessage::from_protocol_message(message), targets);
         }
 
         debug!("{:?} // Broadcasted commit consensus message {:?}",
@@ -162,11 +160,11 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
 
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
-                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST, LP>> {}
+                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {}
 
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
-                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: Node<PBFT<D, ST, LP>> {}
+                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {}
 }
 
 impl<D, ST, LP> ReplicaAccessory<D, ST, LP>
@@ -179,7 +177,7 @@ impl<D, ST, LP> ReplicaAccessory<D, ST, LP>
         }
     }
 
-    fn take_speculative_commits(&self) -> BTreeMap<NodeId, StoredSerializedNetworkMessage<PBFT<D, ST, LP>>> {
+    fn take_speculative_commits(&self) -> BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP>>> {
         let mut map = self.speculative_commits.lock().unwrap();
         std::mem::replace(&mut *map, BTreeMap::new())
     }
@@ -188,7 +186,7 @@ impl<D, ST, LP> ReplicaAccessory<D, ST, LP>
 
 #[inline]
 fn valid_spec_commits<D, ST, LP>(
-    speculative_commits: &BTreeMap<NodeId, StoredSerializedNetworkMessage<PBFT<D, ST, LP>>>,
+    speculative_commits: &BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP>>>,
     node_id: NodeId,
     seq_no: SeqNo,
     view: &ViewInfo,
@@ -212,7 +210,7 @@ fn valid_spec_commits<D, ST, LP>(
 
     speculative_commits
         .values()
-        .map(|stored| match stored.message().original().deref_system() {
+        .map(|stored| match stored.message().original() {
             SystemMessage::ProtocolMessage(protocol) => {
                 match protocol.deref() {
                     PBFTMessage::Consensus(consensus) => consensus,
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 10f6e088..d75ec271 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -2,21 +2,23 @@ use std::collections::VecDeque;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 use std::time::Instant;
+
 use chrono::Utc;
 use log::{debug, info, warn};
-use atlas_common::crypto::hash::Digest;
+
 use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
-use atlas_communication::Node;
+use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::ClientRqInfo;
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, OperationMode};
-use atlas_execution::serialize::ApplicationData;
+use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog};
 use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
-use atlas_metrics::metrics::{metric_duration};
+use atlas_execution::serialize::ApplicationData;
+use atlas_metrics::metrics::metric_duration;
+
 use crate::bft::consensus::accessory::{AccessoryConsensus, ConsensusDecisionAccessory};
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
@@ -267,8 +269,8 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
                                synchronizer: &Synchronizer<D>,
                                timeouts: &Timeouts,
                                log: &mut Log<D, PL>,
-                               node: &NT) -> Result<DecisionStatus>
-        where NT: Node<PBFT<D, ST, LP>>,
+                               node: &Arc<NT>) -> Result<DecisionStatus>
+        where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
               PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let view = synchronizer.view();
 
@@ -395,7 +397,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
                         self.node_id, stored_msg.message(), stored_msg.header().from(), received);
 
                     self.accessory.handle_partial_pre_prepare(&self.message_log,
-                                                              &view, stored_msg.clone(), node);
+                                                              &view, stored_msg.clone(), &**node);
 
                     DecisionPhase::PrePreparing(received)
                 };
@@ -468,7 +470,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
                     let current_digest = self.message_log.current_digest().unwrap();
 
                     self.accessory.handle_preparing_quorum(&self.message_log, &view,
-                                                           stored_msg.clone(), node);
+                                                           stored_msg.clone(), &**node);
 
                     self.message_queue.signal();
 
@@ -480,7 +482,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
                         self.node_id, stored_msg.message().sequence_number(), header.from(), received);
 
                     self.accessory.handle_preparing_no_quorum(&self.message_log, &view,
-                                                              stored_msg.clone(), node);
+                                                              stored_msg.clone(), &**node);
 
                     DecisionPhase::Preparing(received)
                 };
@@ -542,7 +544,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
                     self.consensus_metrics.commit_quorum_recvd();
 
                     self.accessory.handle_committing_quorum(&self.message_log, &view,
-                                                            stored_msg.clone(), node);
+                                                            stored_msg.clone(), &**node);
 
                     Ok(DecisionStatus::Decided)
                 } else {
@@ -552,7 +554,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
                     self.phase = DecisionPhase::Committing(received);
 
                     self.accessory.handle_committing_no_quorum(&self.message_log, &view,
-                                                               stored_msg.clone(), node);
+                                                               stored_msg.clone(), &**node);
 
                     Ok(DecisionStatus::Deciding)
                 };
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 3bf41747..0ee037d8 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -1,42 +1,40 @@
-pub mod decision;
-pub mod accessory;
-
 use std::cmp::Reverse;
 use std::collections::{BinaryHeap, BTreeMap, BTreeSet, VecDeque};
-use std::iter;
 use std::sync::{Arc, Mutex};
 use std::sync::atomic::{AtomicBool, Ordering};
-use chrono::Utc;
+
 use either::Either;
 use event_listener::Event;
 use log::{debug, error, info, trace, warn};
+
 use atlas_common::error::*;
-use atlas_common::crypto::hash::Digest;
-use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{InvalidSeqNo, Orderable, SeqNo, tbo_advance_message_queue, tbo_advance_message_queue_return, tbo_queue_message};
+use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_advance_message_queue_return, tbo_queue_message};
 use atlas_communication::message::{Header, StoredMessage};
-use atlas_communication::Node;
-use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::ApplicationData;
+use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage, SystemMessage};
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
+use atlas_execution::ExecutorHandle;
+use atlas_execution::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_increment;
-use crate::bft;
-use crate::bft::consensus::decision::{ConsensusDecision, DecisionPhase, DecisionPollStatus, DecisionStatus, MessageQueue};
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::deciding_log::{CompletedBatch, DecidingLog};
-use crate::bft::msg_log::decisions::{DecisionLog, IncompleteProof, Proof, StoredConsensusMessage};
+
 use crate::bft::{PBFT, SysMsg};
+use crate::bft::consensus::decision::{ConsensusDecision, DecisionPollStatus, DecisionStatus, MessageQueue};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::OPERATIONS_PROCESSED_ID;
+use crate::bft::msg_log::decided_log::Log;
+use crate::bft::msg_log::deciding_log::CompletedBatch;
+use crate::bft::msg_log::decisions::{DecisionLog, IncompleteProof, Proof};
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer};
 use crate::bft::sync::view::ViewInfo;
 
+pub mod decision;
+pub mod accessory;
+
 #[derive(Debug, Clone)]
 /// Status returned from processing a consensus message.
 pub enum ConsensusStatus {
@@ -371,8 +369,8 @@ impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
                                    synchronizer: &Synchronizer<D>,
                                    timeouts: &Timeouts,
                                    log: &mut Log<D, PL>,
-                                   node: &NT) -> Result<ConsensusStatus>
-        where NT: Node<PBFT<D, ST, LP>>,
+                                   node: &Arc<NT>) -> Result<ConsensusStatus>
+        where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
               PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let message_seq = message.sequence_number();
 
@@ -806,8 +804,8 @@ impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
         synchronizer: &Synchronizer<D>,
         timeouts: &Timeouts,
         log: &mut Log<D, PL>,
-        node: &NT,
-    ) where NT: Node<PBFT<D, ST, LP>>,
+        node: &Arc<NT>,
+    ) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
             PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let view = synchronizer.view();
         //Prepare the algorithm as we are already entering this phase
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index 24937173..2d349d08 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -8,23 +8,18 @@
 
 use std::io::{Read, Write};
 use std::marker::PhantomData;
-use std::sync::Arc;
-use bytes::Bytes;
-use atlas_common::error::*;
 
 #[cfg(feature = "serialize_serde")]
 use ::serde::{Deserialize, Serialize};
-use atlas_common::crypto::hash::{Context, Digest};
-use atlas_common::globals::ReadOnly;
-use atlas_communication::message::StoredMessage;
-use atlas_communication::Node;
+use bytes::Bytes;
+
+use atlas_common::error::*;
 use atlas_communication::serialize::Serializable;
-use atlas_core::ordering_protocol::{ProtocolMessage, SerProof, SerProofMetadata};
 use atlas_core::persistent_log::PersistableOrderProtocol;
-use atlas_execution::serialize::ApplicationData;
 use atlas_core::serialize::{OrderingProtocolMessage, StatefulOrderProtocolMessage};
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
-use crate::bft::{PBFT};
+use atlas_execution::serialize::ApplicationData;
+
+use crate::bft::message::{ConsensusMessage, PBFTMessage};
 use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata};
 use crate::bft::sync::view::ViewInfo;
 
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index b7860ef0..a769f24e 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -20,7 +20,7 @@ use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
-use atlas_communication::Node;
+use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::serialize::Serializable;
 use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
@@ -83,7 +83,7 @@ pub struct PBFTOrderProtocol<D, ST, LP, NT, PL>
         D: ApplicationData + 'static,
         ST: StateTransferMessage + 'static,
         LP: LogTransferMessage + 'static,
-        NT: Node<PBFT<D, ST, LP>> + 'static,
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
         PL: Clone {
     // What phase of the consensus algorithm are we currently executing
     phase: ConsensusPhase,
@@ -115,7 +115,7 @@ pub struct PBFTOrderProtocol<D, ST, LP, NT, PL>
 }
 
 impl<D, ST, LP, NT, PL> Orderable for PBFTOrderProtocol<D, ST, LP, NT, PL> where D: 'static + ApplicationData,
-                                                                                 NT: 'static + Node<PBFT<D, ST, LP>>,
+                                                                                 NT: 'static + ProtocolNetworkNode<PBFT<D, ST, LP>>,
                                                                                  ST: 'static + StateTransferMessage,
                                                                                  LP: 'static + LogTransferMessage,
                                                                                  PL: Clone {
@@ -128,7 +128,7 @@ impl<D, ST, LP, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST,
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: Node<PBFT<D, ST, LP>> + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
           PL: Clone {
     type Serialization = PBFTConsensus<D>;
     type Config = PBFTConfig<D, ST>;
@@ -266,7 +266,7 @@ impl<D, ST, LP, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST,
 impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: ApplicationData + 'static,
                                                                    ST: StateTransferMessage + 'static,
                                                                    LP: LogTransferMessage + 'static,
-                                                                   NT: Node<PBFT<D, ST, LP>> + 'static,
+                                                                   NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
                                                                    PL: Clone {
     fn initialize_protocol(config: PBFTConfig<D, ST>, args: OrderingProtocolArgs<D, NT, PL>,
                            initial_state: Option<DecisionLog<D::Request>>) -> Result<Self> {
@@ -347,7 +347,7 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
                     &mut self.message_log,
                     &self.timeouts,
                     &mut self.consensus,
-                    &*self.node,
+                    &self.node,
                 );
 
                 self.switch_phase(ConsensusPhase::NormalPhase);
@@ -463,7 +463,7 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
                     &mut self.message_log,
                     &self.pre_processor,
                     &mut self.consensus,
-                    &*self.node,
+                    &self.node,
                 );
 
                 self.synchronizer.signal();
@@ -506,7 +506,7 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
             &self.synchronizer,
             &self.timeouts,
             &mut self.message_log,
-            &*self.node,
+            &self.node,
         )?;
 
         match status {
@@ -567,7 +567,7 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
             &mut self.message_log,
             &self.pre_processor,
             &mut self.consensus,
-            &*self.node,
+            &self.node,
         );
 
         self.synchronizer.signal();
@@ -607,7 +607,7 @@ impl<D, ST, LP, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: Node<PBFT<D, ST, LP>> + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
           PL: Clone {
     type StateSerialization = PBFTConsensus<D>;
 
@@ -671,7 +671,7 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: Node<PBFT<D, ST, LP>> + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
           PL: Clone {
     pub(crate) fn switch_phase(&mut self, new_phase: ConsensusPhase) {
         info!("{:?} // Switching from phase {:?} to phase {:?}", self.node.id(), self.phase, new_phase);
@@ -735,7 +735,7 @@ impl<D, ST, LP, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: Node<PBFT<D, ST, LP>>, PL: Clone {
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP>>, PL: Clone {
     fn message_types() -> Vec<&'static str> {
         vec![
             CF_PRE_PREPARES,
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index 811edb02..616e84f7 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -1,27 +1,23 @@
-use std::mem::size_of;
 use std::sync::Arc;
+
 use log::error;
-use atlas_common::crypto::hash::{Context, Digest};
 
 use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::StoredMessage;
-use atlas_communication::Node;
-use atlas_core::ordering_protocol::{DecisionInformation, ExecutionResult, ProtocolConsensusDecision};
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog, OperationMode};
+use atlas_core::ordering_protocol::{DecisionInformation, ProtocolConsensusDecision};
+use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::serialize::StateTransferMessage;
-use atlas_execution::app::{Request, UpdateBatch};
+use atlas_execution::app::UpdateBatch;
 use atlas_execution::serialize::ApplicationData;
-use atlas_core::state_transfer::Checkpoint;
 
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
+use crate::bft::message::{ConsensusMessageKind, PBFTMessage};
 use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::msg_log::{operation_key};
-use crate::bft::msg_log::decisions::{CollectData, DecisionLog, Proof, ProofMetadata};
-use crate::bft::msg_log::deciding_log::{CompletedBatch, DecidingLog};
-use crate::bft::{PBFT, PBFTOrderProtocol};
+use crate::bft::msg_log::operation_key;
+use crate::bft::msg_log::deciding_log::CompletedBatch;
+use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata};
 use crate::bft::sync::view::ViewInfo;
 
 /// The log of decisions that have already been processed by the consensus
diff --git a/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs
index ba219b35..46dc2308 100644
--- a/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs
@@ -1,13 +1,14 @@
 use std::marker::PhantomData;
-use std::sync::{atomic::AtomicBool, Arc};
+use std::sync::{Arc, atomic::AtomicBool};
+
 use atlas_common::channel;
 use atlas_common::channel::{ChannelSyncRx, ChannelSyncTx};
-use atlas_communication::message::StoredMessage;
-use atlas_communication::Node;
+use atlas_communication::protocol_node::ProtocolNetworkNode;
+use atlas_core::messages::StoredRequestMessage;
+use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
-use atlas_core::messages::{RequestMessage, StoredRequestMessage};
-use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
+
 use crate::bft::PBFT;
 
 pub type BatchType<D: ApplicationData> = Vec<StoredRequestMessage<D::Request>>;
@@ -17,7 +18,7 @@ pub struct FollowerProposer<D, ST, LP, NT>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: Node<PBFT<D, ST, LP>> {
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
     batch_channel: (ChannelSyncTx<BatchType<D>>, ChannelSyncRx<BatchType<D>>),
     //For request execution
     executor_handle: ExecutorHandle<D>,
@@ -42,7 +43,7 @@ impl<D, ST, LP, NT> FollowerProposer<D, ST, LP, NT>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: Node<PBFT<D, ST, LP>> {
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
     pub fn new(
         node: Arc<NT>,
         executor: ExecutorHandle<D>,
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index 5807912a..7562440e 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -1,38 +1,36 @@
-pub mod follower_proposer;
-
-use std::cmp::max;
 use std::collections::BTreeMap;
-use std::marker::PhantomData;
-use std::ops::Div;
-use log::{error, warn, debug, info, trace};
-use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, MutexGuard};
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::thread::JoinHandle;
 use std::time::{Duration, Instant};
 
-use chrono::{DateTime, Utc};
-use atlas_common::channel::{ChannelSyncRx, TryRecvError};
+use log::{debug, error, info, warn};
+
+use atlas_common::channel::TryRecvError;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::threadpool;
-use atlas_communication::message::{NetworkMessage, NetworkMessageKind, StoredMessage};
-use atlas_communication::Node;
-use atlas_execution::app::{Request, UnorderedBatch};
-use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::ApplicationData;
-use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage, SystemMessage};
-use atlas_core::request_pre_processing::{BatchOutput, PreProcessorMessage, PreProcessorOutputMessage};
+use atlas_communication::message::NetworkMessageKind;
+use atlas_communication::protocol_node::ProtocolNetworkNode;
+use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
+use atlas_core::request_pre_processing::{BatchOutput, PreProcessorOutputMessage};
 use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
-use atlas_metrics::metrics::{metric_duration, metric_increment, metric_local_duration_end, metric_local_duration_start, metric_store_count};
+use atlas_execution::app::UnorderedBatch;
+use atlas_execution::ExecutorHandle;
+use atlas_execution::serialize::ApplicationData;
+use atlas_metrics::metrics::{metric_duration, metric_increment, metric_store_count};
+
 use crate::bft::config::ProposerConfig;
 use crate::bft::consensus::ProposerConsensusGuard;
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, ObserverMessage, PBFTMessage};
-use crate::bft::metric::{CLIENT_POOL_BATCH_SIZE_ID, PROPOSER_BATCHES_MADE_ID, PROPOSER_FWD_REQUESTS_ID, PROPOSER_LATENCY_ID, PROPOSER_PROPOSE_TIME_ID, PROPOSER_REQUEST_FILTER_TIME_ID, PROPOSER_REQUEST_PROCESSING_TIME_ID, PROPOSER_REQUEST_TIME_ITERATIONS_ID, PROPOSER_REQUESTS_COLLECTED_ID};
-use crate::bft::observer::{ConnState, MessageType, ObserverHandle};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
+use crate::bft::metric::{CLIENT_POOL_BATCH_SIZE_ID, PROPOSER_BATCHES_MADE_ID, PROPOSER_LATENCY_ID, PROPOSER_PROPOSE_TIME_ID, PROPOSER_REQUEST_PROCESSING_TIME_ID, PROPOSER_REQUEST_TIME_ITERATIONS_ID, PROPOSER_REQUESTS_COLLECTED_ID};
 use crate::bft::PBFT;
 use crate::bft::sync::view::{is_request_in_hash_space, ViewInfo};
-use super::sync::{Synchronizer, AbstractSynchronizer};
+
+use super::sync::{AbstractSynchronizer, Synchronizer};
+
+pub mod follower_proposer;
 
 pub type BatchType<R> = Vec<StoredRequestMessage<R>>;
 
@@ -111,7 +109,7 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
     pub fn start<ST, LP>(self: Arc<Self>) -> JoinHandle<()>
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<PBFT<D, ST, LP>> + 'static {
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static {
         std::thread::Builder::new()
             .name(format!("Proposer thread"))
             .spawn(move || {
@@ -251,7 +249,7 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
         propose: &mut ProposeBuilder<D>,
     ) -> bool where ST: StateTransferMessage + 'static,
                     LP: LogTransferMessage + 'static,
-                    NT: Node<PBFT<D, ST, LP>> {
+                    NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         if !propose.currently_accumulated.is_empty() {
             let current_batch_size = propose.currently_accumulated.len();
 
@@ -322,7 +320,7 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
                                propose: &mut ProposeBuilder<D>, ) -> bool
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<PBFT<D, ST, LP>> {
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
 
         //Now let's deal with ordered requests
         if is_leader {
@@ -381,7 +379,7 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
         mut currently_accumulated: Vec<StoredRequestMessage<D::Request>>,
     ) where ST: StateTransferMessage + 'static,
             LP: LogTransferMessage + 'static,
-            NT: Node<PBFT<D, ST, LP>> {
+            NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         let has_pending_messages = self.consensus_guard.has_pending_view_change_reqs();
 
         let is_view_change_empty = {
@@ -423,7 +421,7 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
 
         let targets = view.quorum_members().iter().copied();
 
-        self.node_ref.broadcast_signed(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), targets);
+        self.node_ref.broadcast_signed(SystemMessage::from_protocol_message(message), targets);
 
         metric_increment(PROPOSER_BATCHES_MADE_ID, Some(1));
     }
diff --git a/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
index 512fb679..2321b3d4 100644
--- a/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
@@ -1,12 +1,9 @@
-use std::{marker::PhantomData, sync::Arc};
-use atlas_common::crypto::hash::Digest;
+use std::{marker::PhantomData};
 use atlas_common::ordering::Orderable;
-use atlas_communication::message::StoredMessage;
 use atlas_core::messages::ClientRqInfo;
-use atlas_execution::app::{Request};
 use atlas_execution::serialize::ApplicationData;
 
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
+use crate::bft::message::{ConsensusMessageKind};
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
 
 pub struct FollowerSynchronizer<D: ApplicationData> {
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 7c1e9b70..ed6014ec 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -7,10 +7,7 @@ use std::{
 };
 use std::cell::Cell;
 use std::fmt::{Debug, Formatter};
-use std::sync::MutexGuard;
-use bytes::BytesMut;
 use either::Either;
-use futures::SinkExt;
 
 use self::{follower_sync::FollowerSynchronizer, replica_sync::ReplicaSynchronizer};
 
@@ -24,7 +21,9 @@ use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_qu
 use atlas_common::{collections, prng};
 use atlas_common::node_id::NodeId;
 use atlas_communication::message::{Header, NetworkMessageKind, StoredMessage, System, WireMessage};
-use atlas_communication::{Node, NodePK, serialize};
+use atlas_communication::{ serialize};
+use atlas_communication::protocol_node::ProtocolNetworkNode;
+use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_communication::serialize::Buf;
 use atlas_execution::app::{Reply, Request};
 use atlas_execution::serialize::ApplicationData;
@@ -505,11 +504,11 @@ impl<D> Synchronizer<D>
         log: &mut Log<D, PL>,
         rq_pre_processor: &RequestPreProcessor<D::Request>,
         consensus: &mut Consensus<D, ST, LP, PL>,
-        node: &NT,
+        node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<PBFT<D, ST, LP>>,
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
               PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         debug!("{:?} // Processing view change message {:?} in phase {:?} from {:?}",
@@ -630,7 +629,7 @@ impl<D> Synchronizer<D>
                 // we have sent our own STOP message
                 if let ProtoPhase::Stopping(_i) = self.phase.get() {
                     return if i > current_view.params().f() {
-                        self.begin_view_change(None, node, timeouts, log);
+                        self.begin_view_change(None, &**node, timeouts, log);
                         SynchronizerStatus::Running
                     } else {
                         self.phase.replace(ProtoPhase::Stopping(i));
@@ -656,7 +655,7 @@ impl<D> Synchronizer<D>
                     match &self.accessory {
                         SynchronizerAccessory::Replica(rep) => {
                             rep.handle_stopping_quorum(self, previous_view, consensus,
-                                                       log, rq_pre_processor, timeouts, node)
+                                                       log, rq_pre_processor, timeouts, &**node)
                         }
                         SynchronizerAccessory::Follower(_) => {}
                     }
@@ -789,7 +788,7 @@ impl<D> Synchronizer<D>
                             let previous_view_ref = previous_view.as_ref().unwrap_or(&current_view);
 
                             let proof = Self::highest_proof(&*collects_guard,
-                                                            previous_view_ref, node);
+                                                            previous_view_ref, &**node);
 
                             info!("{:?} // Highest proof: {:?}", node.id(), proof);
 
@@ -822,25 +821,18 @@ impl<D> Synchronizer<D>
                             }
 
                             let p = rq_pre_processor.collect_all_pending_rqs();
-                            let node_sign = node.pk_crypto().get_key_pair().clone();
+                            let node_sign = node.network_info_provider().get_key_pair().clone();
 
                             //We create the pre-prepare here as we are the new leader,
                             //And we sign it right now
                             let (header, message) = {
-                                let mut buf = Vec::new();
-
                                 info!("{:?} // Forged pre-prepare: {}", node.id(), p.len());
 
                                 let forged_pre_prepare = consensus.forge_propose(p, self);
 
-                                let forged_pre_prepare = NetworkMessageKind::from_system(forged_pre_prepare);
-
-                                let digest = serialize::serialize_digest::<Vec<u8>, PBFT<D, ST, LP>>(
-                                    &forged_pre_prepare,
-                                    &mut buf,
-                                ).unwrap();
+                                let (message, digest) = node.serialize_digest_message(forged_pre_prepare).unwrap();
 
-                                let buf = Buf::from(buf);
+                                let (message, buf) = message.into_inner();
 
                                 let mut prng_state = prng::State::new();
 
@@ -855,7 +847,7 @@ impl<D> Synchronizer<D>
                                     Some(&*node_sign),
                                 ).into_inner();
 
-                                if let PBFTMessage::Consensus(consensus) = forged_pre_prepare.into_system().into_protocol_message() {
+                                if let PBFTMessage::Consensus(consensus) = message.into_protocol_message() {
                                     (h, consensus)
                                 } else {
                                     //This is basically impossible
@@ -880,7 +872,7 @@ impl<D> Synchronizer<D>
                             let targets = NodeId::targets(0..current_view.params().n())
                                 .filter(move |&id| id != node_id);
 
-                            node.broadcast(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), targets);
+                            node.broadcast(SystemMessage::from_protocol_message(message), targets);
 
                             let state = FinalizeState {
                                 curr_cid,
@@ -967,9 +959,9 @@ impl<D> Synchronizer<D>
 
                 // leader has already performed this computation in the
                 // STOP-DATA phase of Mod-SMaRt
-                let signed: Vec<_> = signed_collects::<D, _, _, _>(node, collects);
+                let signed: Vec<_> = signed_collects::<D, _, _, _>(&**node, collects);
 
-                let proof = highest_proof::<D, _, _, _, _>(&current_view, node, signed.iter());
+                let proof = highest_proof::<D, _, _, _, _>(&current_view, &**node, signed.iter());
 
                 let curr_cid = proof
                     .map(|p| p.sequence_number())
@@ -1022,11 +1014,11 @@ impl<D> Synchronizer<D>
         log: &mut Log<D, PL>,
         timeouts: &Timeouts,
         consensus: &mut Consensus<D, ST, LP, PL>,
-        node: &NT,
+        node: &Arc<NT>,
     ) -> Option<()>
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<PBFT<D, ST, LP>>,
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
               PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         let state = self.finalize_state.borrow_mut().take()?;
@@ -1063,7 +1055,7 @@ impl<D> Synchronizer<D>
     )
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<PBFT<D, ST, LP>>,
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         match (self.phase.get(), &timed_out) {
@@ -1148,11 +1140,11 @@ impl<D> Synchronizer<D>
         log: &mut Log<D, PL>,
         timeouts: &Timeouts,
         consensus: &mut Consensus<D, ST, LP, PL>,
-        node: &NT,
+        node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<PBFT<D, ST, LP>>,
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
               PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         let FinalizeState {
@@ -1242,7 +1234,7 @@ impl<D> Synchronizer<D>
                                         node: &NT)
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<PBFT<D, ST, LP>> {
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         match &self.accessory {
             SynchronizerAccessory::Follower(_) => {}
             SynchronizerAccessory::Replica(rep) => {
@@ -1294,7 +1286,7 @@ impl<D> Synchronizer<D>
     ) -> Option<&'a Proof<D::Request>>
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<PBFT<D, ST, LP>>
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP>>
     {
         highest_proof::<D, _, _, _, _>(&view, node, guard.values())
     }
@@ -1529,7 +1521,7 @@ fn signed_collects<D, ST, LP, NT>(
         D: ApplicationData + 'static,
         ST: StateTransferMessage + 'static,
         LP: LogTransferMessage + 'static,
-        NT: Node<PBFT<D, ST, LP>>
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP>>
 {
     collects
         .into_iter()
@@ -1542,7 +1534,7 @@ fn validate_signature<'a, D, M, ST, LP, NT>(node: &'a NT, stored: &'a StoredMess
         D: ApplicationData + 'static,
         ST: StateTransferMessage + 'static,
         LP: LogTransferMessage + 'static,
-        NT: Node<PBFT<D, ST, LP>>
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP>>
 {
     //TODO: Fix this as I believe it will always be false
     let wm = match WireMessage::from_header(*stored.header()) {
@@ -1556,7 +1548,7 @@ fn validate_signature<'a, D, M, ST, LP, NT>(node: &'a NT, stored: &'a StoredMess
 
     // check if we even have the public key of the node that claims
     // to have sent this particular message
-    let key = match node.pk_crypto().get_public_key(&stored.header().from()) {
+    let key = match node.network_info_provider().get_public_key(&stored.header().from()) {
         Some(k) => k,
         None => {
             error!("{:?} // Failed to get public key for node {:?}", node.id(), stored.header().from());
@@ -1578,7 +1570,7 @@ fn highest_proof<'a, D, I, ST, LP, NT>(
         I: Iterator<Item=&'a StoredMessage<ViewChangeMessage<D::Request>>>,
         ST: StateTransferMessage + 'static,
         LP: LogTransferMessage + 'static,
-        NT: Node<PBFT<D, ST, LP>>
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP>>
 {
     collect_data(collects)
         // fetch proofs
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index fd369c1b..3371ec12 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -6,19 +6,14 @@
 use std::cell::Cell;
 
 use std::marker::PhantomData;
-use std::sync::Arc;
-use std::sync::atomic::AtomicBool;
 
 use std::time::{Duration, Instant};
 use log::{debug, error, info};
 use atlas_common::collections;
-use atlas_common::collections::ConcurrentHashMap;
-use atlas_common::crypto::hash::Digest;
 use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_communication::message::{NetworkMessageKind, StoredMessage, System};
-use atlas_communication::{Node};
-use atlas_execution::app::{Request};
+use atlas_common::ordering::{Orderable};
+use atlas_communication::message::{NetworkMessageKind};
+use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, RequestMessage, StoredRequestMessage, SystemMessage};
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
@@ -75,7 +70,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     )
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: Node<PBFT<D, ST, LP>>,
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>> {
         // NOTE:
         // - install new view (i.e. update view seq no) (Done in the synchronizer)
@@ -109,7 +104,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
             ViewChangeMessageKind::StopData(collect),
         ));
 
-        node.send_signed(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), current_leader, true);
+        node.send_signed(SystemMessage::from_protocol_message(message), current_leader, true);
     }
 
     /// Start a new view change
@@ -123,7 +118,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         timed_out: Option<Vec<StoredRequestMessage<D::Request>>>,
     ) where ST: StateTransferMessage + 'static,
             LP: LogTransferMessage + 'static,
-            NT: Node<PBFT<D, ST, LP>> {
+            NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         // stop all timers
         self.unwatch_all_requests(timeouts);
 
@@ -146,7 +141,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
 
         let targets = NodeId::targets(0..current_view.params().n());
 
-        node.broadcast(NetworkMessageKind::from_system(SystemMessage::from_protocol_message(message)), targets);
+        node.broadcast(SystemMessage::from_protocol_message(message), targets);
     }
 
     /// Watch a vector of requests received
@@ -343,10 +338,10 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         node: &NT,
     ) where ST: StateTransferMessage + 'static,
             LP: LogTransferMessage + 'static,
-            NT: Node<PBFT<D, ST, LP>> {
+            NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         let message = SystemMessage::ForwardedRequestMessage(ForwardedRequestsMessage::new(timed_out));
         let targets = NodeId::targets(0..base_sync.view().params().n());
-        node.broadcast(NetworkMessageKind::from_system(message), targets);
+        node.broadcast(message, targets);
     }
 
     /// Obtain the requests that we know have timed out so we can send out a stop message

From 4701c77e0ff6bb8ad1e1150a9e338ed22bf1caa1 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Tue, 18 Jul 2023 19:04:37 +0100
Subject: [PATCH 18/80] Made the protocols use the quorum members instead of
 directly referring to the node ids

---
 .../bft/consensus/accessory/replica/mod.rs    |  8 +--
 febft-pbft-consensus/src/bft/mod.rs           |  2 +-
 febft-pbft-consensus/src/bft/proposer/mod.rs  | 16 +++--
 febft-pbft-consensus/src/bft/sync/mod.rs      | 53 ++++++++++------
 .../src/bft/sync/replica_sync/mod.rs          | 11 ++--
 febft-pbft-consensus/src/bft/sync/view/mod.rs | 61 +++++++++++++------
 febft-state-transfer/src/lib.rs               |  4 +-
 7 files changed, 101 insertions(+), 54 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index 5488deb0..a775d51b 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -111,9 +111,9 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
             ConsensusMessageKind::Prepare(current_digest),
         ));
 
-        let targets = NodeId::targets(0..view.params().n());
+        let targets = view.quorum_members().clone();
 
-        node.broadcast_signed(SystemMessage::from_protocol_message(message), targets);
+        node.broadcast_signed(SystemMessage::from_protocol_message(message), targets.into_iter());
     }
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
@@ -147,9 +147,9 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
             debug!("{:?} // Broadcasting commit consensus message {:?}",
                         node_id, message);
 
-            let targets = NodeId::targets(0..view.params().n());
+            let targets = view.quorum_members().clone();
 
-            node.broadcast_signed(SystemMessage::from_protocol_message(message), targets);
+            node.broadcast_signed(SystemMessage::from_protocol_message(message), targets.into_iter());
         }
 
         debug!("{:?} // Broadcasted commit consensus message {:?}",
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index a769f24e..40ffe324 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -290,7 +290,7 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
 
         let dec_log = initialize_decided_log::<D, PL>(node_id, persistent_log, initial_state)?;
 
-        let proposer = Proposer::<D, NT>::new(node.clone(), batch_input, sync.clone(), timeouts.clone(),
+        let proposer = Proposer::<D, NT, RP>::new(node.clone(), batch_input, sync.clone(), timeouts.clone(),
                                               executor.clone(), consensus_guard.clone(),
                                               proposer_config);
 
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index 7562440e..4b48a6cb 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -13,6 +13,7 @@ use atlas_common::threadpool;
 use atlas_communication::message::NetworkMessageKind;
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
+use atlas_core::reconfiguration_protocol::ReconfigurationProtocol;
 use atlas_core::request_pre_processing::{BatchOutput, PreProcessorOutputMessage};
 use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
@@ -77,7 +78,8 @@ impl<D> ProposeBuilder<D> where D: ApplicationData {
 ///The size of the batch channel
 const BATCH_CHANNEL_SIZE: usize = 128;
 
-impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
+impl<D, NT> Proposer<D, NT>
+    where D: ApplicationData + 'static, {
     pub fn new(
         node: Arc<NT>,
         batch_input: BatchOutput<D::Request>,
@@ -87,6 +89,7 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
         consensus_guard: Arc<ProposerConsensusGuard>,
         proposer_config: ProposerConfig,
     ) -> Arc<Self> {
+
         let ProposerConfig {
             target_batch_size, max_batch_size, batch_timeout
         } = proposer_config;
@@ -377,9 +380,10 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
         seq: SeqNo,
         view: &ViewInfo,
         mut currently_accumulated: Vec<StoredRequestMessage<D::Request>>,
-    ) where ST: StateTransferMessage + 'static,
-            LP: LogTransferMessage + 'static,
-            NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+    ) where
+        ST: StateTransferMessage + 'static,
+        LP: LogTransferMessage + 'static,
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         let has_pending_messages = self.consensus_guard.has_pending_view_change_reqs();
 
         let is_view_change_empty = {
@@ -419,9 +423,9 @@ impl<D, NT> Proposer<D, NT> where D: ApplicationData + 'static {
             ConsensusMessageKind::PrePrepare(currently_accumulated),
         ));
 
-        let targets = view.quorum_members().iter().copied();
+        let targets = view.quorum_members().clone();
 
-        self.node_ref.broadcast_signed(SystemMessage::from_protocol_message(message), targets);
+        self.node_ref.broadcast_signed(SystemMessage::from_protocol_message(message), targets.into_iter());
 
         metric_increment(PROPOSER_BATCHES_MADE_ID, Some(1));
     }
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index ed6014ec..0e475eb2 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -3,45 +3,42 @@ use std::{
     cmp::Ordering,
     collections::VecDeque,
     sync::{Arc, Mutex},
-    time::{Duration, Instant},
+    time::Duration,
 };
 use std::cell::Cell;
 use std::fmt::{Debug, Formatter};
-use either::Either;
-
-use self::{follower_sync::FollowerSynchronizer, replica_sync::ReplicaSynchronizer};
 
+use either::Either;
 use intmap::IntMap;
 use log::{debug, error, info, warn};
-
 #[cfg(feature = "serialize_serde")]
 use serde::{Deserialize, Serialize};
-use atlas_common::crypto::hash::Digest;
-use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_queue_message, tbo_pop_message, InvalidSeqNo};
+
 use atlas_common::{collections, prng};
+use atlas_common::crypto::hash::Digest;
+use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
-use atlas_communication::message::{Header, NetworkMessageKind, StoredMessage, System, WireMessage};
-use atlas_communication::{ serialize};
+use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_pop_message, tbo_queue_message};
+use atlas_communication::message::{Header, StoredMessage, WireMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
-use atlas_communication::serialize::Buf;
-use atlas_execution::app::{Reply, Request};
-use atlas_execution::serialize::ApplicationData;
-use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, RequestMessage, StoredRequestMessage, SystemMessage};
+use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
+use atlas_core::request_pre_processing::RequestPreProcessor;
 use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
+use atlas_execution::serialize::ApplicationData;
+
+use crate::bft::PBFT;
 use crate::bft::consensus::Consensus;
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, FwdConsensusMessage, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
+use crate::bft::message::{ConsensusMessageKind, FwdConsensusMessage, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::msg_log::decided_log::Log;
 use crate::bft::msg_log::decisions::{CollectData, Proof, StoredConsensusMessage, ViewDecisionPair};
-use crate::bft::{PBFT, PBFTOrderProtocol};
-
 use crate::bft::sync::view::ViewInfo;
 
+use self::{follower_sync::FollowerSynchronizer, replica_sync::ReplicaSynchronizer};
 
 pub mod follower_sync;
 pub mod replica_sync;
@@ -442,6 +439,25 @@ impl<D> Synchronizer<D>
         })
     }
 
+    /// Initialize a new `Synchronizer` with the given quorum members.
+    pub fn initialize_with_quorum(seq_no: SeqNo, quorum_members: Vec<NodeId>, timeout_dur: Duration) -> Result<Arc<Self>> {
+
+        let n = quorum_members.len();
+
+        let f = (n - 1) / 3;
+
+        let view_info = ViewInfo::new(seq_no, n ,f)?;
+
+        Ok(Arc::new(Self {
+            phase: Cell::new(ProtoPhase::Init),
+            tbo: Mutex::new(TboQueue::new(view_info)),
+            stopped: RefCell::new(Default::default()),
+            collects: Mutex::new(Default::default()),
+            finalize_state: RefCell::new(None),
+            accessory: SynchronizerAccessory::Replica(ReplicaSynchronizer::new(timeout_dur)),
+        }))
+    }
+
     fn previous_view(&self) -> Option<ViewInfo> { self.tbo.lock().unwrap().previous_view().clone() }
 
     /// Signal this `TboQueue` that it may be able to extract new
@@ -869,7 +885,8 @@ impl<D> Synchronizer<D>
                             ));
 
                             let node_id = node.id();
-                            let targets = NodeId::targets(0..current_view.params().n())
+
+                            let targets = current_view.quorum_members().clone().into_iter()
                                 .filter(move |&id| id != node_id);
 
                             node.broadcast(SystemMessage::from_protocol_message(message), targets);
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 3371ec12..2d6bd9bc 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -139,9 +139,9 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
             ViewChangeMessageKind::Stop(requests),
         ));
 
-        let targets = NodeId::targets(0..current_view.params().n());
+        let targets = current_view.quorum_members().clone();
 
-        node.broadcast(SystemMessage::from_protocol_message(message), targets);
+        node.broadcast(SystemMessage::from_protocol_message(message), targets.into_iter());
     }
 
     /// Watch a vector of requests received
@@ -340,8 +340,11 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
             LP: LogTransferMessage + 'static,
             NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
         let message = SystemMessage::ForwardedRequestMessage(ForwardedRequestsMessage::new(timed_out));
-        let targets = NodeId::targets(0..base_sync.view().params().n());
-        node.broadcast(message, targets);
+        let view = base_sync.view();
+
+        let targets = view.quorum_members().clone();
+
+        node.broadcast(message, targets.into_iter());
     }
 
     /// Obtain the requests that we know have timed out so we can send out a stop message
diff --git a/febft-pbft-consensus/src/bft/sync/view/mod.rs b/febft-pbft-consensus/src/bft/sync/view/mod.rs
index 00dad91e..f0b2736d 100644
--- a/febft-pbft-consensus/src/bft/sync/view/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/view/mod.rs
@@ -22,8 +22,8 @@ use atlas_core::serialize::NetworkView;
 pub struct ViewInfo {
     // The seq no of the view
     seq: SeqNo,
-    // The ids of the replicas that are currently a part of the quorum
-    quorum: Vec<NodeId>,
+    // The set of nodes in the view
+    quorum_members: Vec<NodeId>,
     // The set of leaders
     leader_set: Vec<NodeId>,
     //TODO: Do we need this? Higher cost of cloning
@@ -48,6 +48,10 @@ impl NetworkView for ViewInfo {
         self.params().quorum()
     }
 
+    fn quorum_members(&self) -> &Vec<NodeId> {
+        &self.quorum_members
+    }
+
     fn f(&self) -> usize {
         self.params().f()
     }
@@ -81,7 +85,33 @@ impl ViewInfo {
 
         Ok(ViewInfo {
             seq,
-            quorum: quorum_members,
+            quorum_members,
+            leader_set,
+            leader_hash_space_division: division,
+            params,
+        })
+    }
+
+    /// Creates a new instance of `ViewInfo`, from a given list of quorum members
+    pub fn from_quorum(seq: SeqNo, quorum_members: Vec<NodeId>) -> Result<Self> {
+        let n = quorum_members.len();
+        let f = (n - 1) / 3;
+
+        let params = SystemParams::new(n, f)?;
+
+        let destined_leader = quorum_members[(usize::from(seq)) % n];
+
+        let mut leader_set = vec![destined_leader];
+
+        for i in 1..LEADER_COUNT {
+            leader_set.push(quorum_members[(usize::from(seq) + i) % n]);
+        }
+
+        let division = calculate_hash_space_division(&leader_set);
+
+        Ok(ViewInfo {
+            seq,
+            quorum_members,
             leader_set,
             leader_hash_space_division: division,
             params,
@@ -105,7 +135,7 @@ impl ViewInfo {
 
         Ok(ViewInfo {
             seq,
-            quorum: quorum_participants,
+            quorum_members: quorum_participants,
             leader_set,
             leader_hash_space_division: division,
             params,
@@ -130,7 +160,7 @@ impl ViewInfo {
 
     /// Returns the primary of the current view.
     pub fn leader(&self) -> NodeId {
-        self.quorum[usize::from(self.seq) % self.params.n()]
+        self.quorum_members[usize::from(self.seq) % self.params.n()]
     }
 
     /// The set of leaders for this view.
@@ -140,14 +170,13 @@ impl ViewInfo {
 
     /// The quorum members for this view
     pub fn quorum_members(&self) -> &Vec<NodeId> {
-        &self.quorum
+        &self.quorum_members
     }
 
     // Get the division of hash spaces for this view
     pub fn hash_space_division(&self) -> &BTreeMap<NodeId, (Vec<u8>, Vec<u8>)> {
         &self.leader_hash_space_division
     }
-
 }
 
 /// Get the division of hash spaces for a given leader_set
@@ -159,7 +188,6 @@ fn calculate_hash_space_division(leader_set: &Vec<NodeId>) -> BTreeMap<NodeId, (
     let mut slice_for_leaders = BTreeMap::new();
 
     slices.into_iter().enumerate().for_each(|(leader_id, slice)| {
-
         let leader = leader_set[leader_id].clone();
 
         slice_for_leaders.insert(leader, slice);
@@ -170,7 +198,6 @@ fn calculate_hash_space_division(leader_set: &Vec<NodeId>) -> BTreeMap<NodeId, (
 
 /// Check if a given requests is within a given hash space
 pub fn is_request_in_hash_space(rq: &Digest, hash_space: &(Vec<u8>, Vec<u8>)) -> bool {
-
     let start = &hash_space.0;
     let end = &hash_space.1;
 
@@ -190,7 +217,7 @@ fn divide_hash_space(size_bytes: usize, count: usize) -> Vec<(Vec<u8>, Vec<u8>)>
 
     let mut start = BigUint::zero();
 
-    let last_hash : Vec<u8> = iter::repeat(0xFF).take(size_bytes).collect();
+    let last_hash: Vec<u8> = iter::repeat(0xFF).take(size_bytes).collect();
 
     //Byte order does not matter, it's all 1s
     let end = BigUint::from_bytes_be(&last_hash[..]);
@@ -202,9 +229,8 @@ fn divide_hash_space(size_bytes: usize, count: usize) -> Vec<(Vec<u8>, Vec<u8>)>
 
     // Get the slices
     for i in 1..=count {
-
         let slice_start = start.to_bytes_be();
-        
+
         start = start.add(increment.clone());
 
         let slice_end = if i == count {
@@ -232,18 +258,17 @@ mod view_tests {
     fn test_hash_space_partition() {
         use super::*;
 
-        const TESTS : usize = 10000;
+        const TESTS: usize = 10000;
 
         let view_info = ViewInfo::new(SeqNo::ZERO, 4, 1).unwrap();
 
         let division = calculate_hash_space_division(view_info.leader_set());
 
-        let mut digest_vec : [u8; Digest::LENGTH] = [0; Digest::LENGTH];
+        let mut digest_vec: [u8; Digest::LENGTH] = [0; Digest::LENGTH];
 
         let mut rng = rand::rngs::SmallRng::seed_from_u64(812679233723);
 
         for i in 0..TESTS {
-
             rng.fill_bytes(&mut digest_vec);
 
             let digest = Digest::from_bytes(&digest_vec).unwrap();
@@ -252,20 +277,18 @@ mod view_tests {
 
             for leader in view_info.leader_set() {
                 if is_request_in_hash_space(&digest, division.get(leader).unwrap()) {
-                    count+=1;
+                    count += 1;
                 }
             }
 
             assert_eq!(count, 1, "The digest {:?} was found in {} hash spaces", digest, count);
         }
     }
-
-
 }
 
 impl Debug for ViewInfo {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(f, "Seq: {:?}, quorum: {:?}, primary: {:?}, leader_set: {:?}, params: {:?}",
-               self.seq, self.quorum, self.leader(), self.leader_set, self.params)
+               self.seq, self.quorum_members, self.leader(), self.leader_set, self.params)
     }
 }
\ No newline at end of file
diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 01e58734..2aec064c 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -954,7 +954,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
             CstMessageKind::RequestStateCid,
         );
 
-        let targets = NodeId::targets(0..view.n());
+        let targets = view.quorum_members().clone().into_iter().filter(|id| *id != self.node.id());
 
         self.node.broadcast(SystemMessage::from_state_transfer_message(message), targets);
     }
@@ -986,7 +986,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         //TODO: Maybe attempt to use followers to rebuild state and avoid
         // Overloading the replicas
         let message = CstMessage::new(cst_seq, CstMessageKind::RequestState);
-        let targets = NodeId::targets(0..view.n()).filter(|id| *id != self.node.id());
+        let targets = view.quorum_members().clone().into_iter().filter(|id| *id != self.node.id());
 
         self.node.broadcast(SystemMessage::from_state_transfer_message(message), targets);
     }

From 2624484ff0a04f732828d0976c8948a0fdd18d2a Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 19 Jul 2023 17:25:57 +0100
Subject: [PATCH 19/80] Added necessary types for new messages.

---
 febft-pbft-consensus/src/bft/config/mod.rs    |  12 +-
 .../src/bft/consensus/accessory/mod.rs        |  47 +++--
 .../bft/consensus/accessory/replica/mod.rs    |  42 ++--
 .../src/bft/consensus/decision/mod.rs         |  44 +++--
 febft-pbft-consensus/src/bft/consensus/mod.rs |  55 +++---
 febft-pbft-consensus/src/bft/message/mod.rs   |  41 ++--
 .../src/bft/message/serialize/mod.rs          |  15 +-
 febft-pbft-consensus/src/bft/mod.rs           | 183 +++++++++---------
 .../src/bft/msg_log/decided_log/mod.rs        |  43 ++--
 .../src/bft/msg_log/deciding_log/mod.rs       |  30 +--
 .../src/bft/msg_log/decisions/mod.rs          |  10 +-
 .../src/bft/proposer/follower_proposer/mod.rs |  14 +-
 febft-pbft-consensus/src/bft/proposer/mod.rs  |  28 +--
 .../src/bft/sync/follower_sync/mod.rs         |   2 +-
 febft-pbft-consensus/src/bft/sync/mod.rs      | 156 +++++++--------
 .../src/bft/sync/replica_sync/mod.rs          |  62 +++---
 16 files changed, 416 insertions(+), 368 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/config/mod.rs b/febft-pbft-consensus/src/bft/config/mod.rs
index 8364abc5..dbf4aaa7 100644
--- a/febft-pbft-consensus/src/bft/config/mod.rs
+++ b/febft-pbft-consensus/src/bft/config/mod.rs
@@ -3,16 +3,17 @@ use std::time::Duration;
 
 use atlas_common::node_id::NodeId;
 use atlas_core::followers::FollowerHandle;
-use atlas_core::serialize::{OrderingProtocolMessage, StateTransferMessage};
+use atlas_core::serialize::{OrderingProtocolMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_execution::serialize::ApplicationData;
 
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::sync::view::ViewInfo;
 
-pub struct PBFTConfig<D: ApplicationData, ST> {
+pub struct PBFTConfig<D, ST, RP>
+    where D: ApplicationData, RP: ReconfigurationProtocolMessage + 'static {
     pub node_id: NodeId,
     // pub observer_handle: ObserverHandle,
-    pub follower_handle: Option<FollowerHandle<PBFTConsensus<D>>>,
+    pub follower_handle: Option<FollowerHandle<PBFTConsensus<D, RP>>>,
     pub view: ViewInfo,
     pub timeout_dur: Duration,
     pub proposer_config: ProposerConfig,
@@ -21,9 +22,10 @@ pub struct PBFTConfig<D: ApplicationData, ST> {
 }
 
 impl<D: ApplicationData + 'static,
-    ST: StateTransferMessage + 'static> PBFTConfig<D, ST> {
+    ST: StateTransferMessage + 'static,
+    RP: ReconfigurationProtocolMessage + 'static> PBFTConfig<D, ST, RP> {
     pub fn new(node_id: NodeId,
-               follower_handle: Option<FollowerHandle<PBFTConsensus<D>>>,
+               follower_handle: Option<FollowerHandle<PBFTConsensus<D, RP>>>,
                view: ViewInfo, timeout_dur: Duration,
                watermark: u32, proposer_config: ProposerConfig) -> Self {
         Self {
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
index db7d6905..c9605169 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
@@ -1,6 +1,6 @@
 use std::sync::Arc;
 use atlas_communication::protocol_node::ProtocolNetworkNode;
-use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_execution::serialize::ApplicationData;
 
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
@@ -11,63 +11,68 @@ use crate::bft::sync::view::ViewInfo;
 
 pub mod replica;
 
-pub enum ConsensusDecisionAccessory<D, ST, LP>
-    where D: ApplicationData + 'static, ST: StateTransferMessage + 'static, LP: LogTransferMessage + 'static {
+pub enum ConsensusDecisionAccessory<D, ST, LP, RP>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static {
     Follower,
-    Replica(ReplicaAccessory<D, ST, LP>),
+    Replica(ReplicaAccessory<D, ST, LP, RP>),
 }
 
-pub trait AccessoryConsensus<D, ST, LP> where D: ApplicationData + 'static,
-                                              ST: StateTransferMessage + 'static,
-                                              LP: LogTransferMessage + 'static {
+pub trait AccessoryConsensus<D, ST, LP, RP> where D: ApplicationData + 'static,
+                                                  ST: StateTransferMessage + 'static,
+                                                  LP: LogTransferMessage + 'static,
+                                                  RP: ReconfigurationProtocolMessage + 'static {
     /// Handle the reception of a pre-prepare message without having completed the pre prepare phase
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>>;
+                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>;
 
     /// Handle the prepare phase having been completed
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static;
+                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static;
 
     /// Handle a prepare message processed during the preparing phase without having
     /// reached a quorum
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>>;
+                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>;
 
     /// Handle a prepare message processed during the prepare phase when a quorum
     /// has been achieved
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
-                                   node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>>;
+                                   node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>;
 
     /// Handle a commit message processed during the preparing phase without having
     /// reached a quorum
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
-                                       node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>>;
+                                       node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>;
 
     /// Handle a commit message processed during the prepare phase when a quorum has been reached
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
-                                    node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>>;
+                                    node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>;
 }
 
-impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D, ST, LP>
+impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ConsensusDecisionAccessory<D, ST, LP, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static {
+          LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static {
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -79,7 +84,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static {
+                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -91,7 +96,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -103,7 +108,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
-                                   node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+                                   node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -115,7 +120,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
-                                       node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+                                       node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -127,7 +132,7 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ConsensusDecisionAccessory<D,
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
-                                    node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+                                    node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index a775d51b..13eb6e5d 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -12,7 +12,7 @@ use atlas_communication::message::{NetworkMessageKind, SerializedMessage, Stored
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::messages::SystemMessage;
-use atlas_core::serialize::{LogTransferMessage, ServiceMsg, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, ServiceMsg, StateTransferMessage};
 use atlas_execution::serialize::ApplicationData;
 
 use crate::bft::consensus::accessory::AccessoryConsensus;
@@ -22,27 +22,29 @@ use crate::bft::msg_log::decisions::StoredConsensusMessage;
 use crate::bft::{PBFT, SysMsg};
 use crate::bft::sync::view::ViewInfo;
 
-pub struct ReplicaAccessory<D, ST, LP>
+pub struct ReplicaAccessory<D, ST, LP, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static {
-    speculative_commits: Arc<Mutex<BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP>>>>>,
+          LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static {
+    speculative_commits: Arc<Mutex<BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP, RP>>>>>,
 }
 
-impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
+impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ReplicaAccessory<D, ST, LP, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static {
+          LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static {
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {}
+                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {}
 
     fn handle_pre_prepare_phase_completed<NT>(&mut self,
                                               deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               _msg: StoredConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static {
+                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static {
         let my_id = node.id();
         let view_seq = view.sequence_number();
         let current_digest = deciding_log.current_digest().unwrap();
@@ -118,18 +120,18 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {}
+                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {}
 
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
-                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         let node_id = node.id();
 
         let seq = deciding_log.sequence_number();
         let current_digest = deciding_log.current_digest().unwrap();
         let speculative_commits = self.take_speculative_commits();
 
-        if valid_spec_commits::<D, ST, LP>(&speculative_commits, node_id, seq, view) {
+        if valid_spec_commits::<D, ST, LP, RP>(&speculative_commits, node_id, seq, view) {
             for (_, msg) in speculative_commits.iter() {
                 debug!("{:?} // Broadcasting speculative commit message (total of {} messages) to {} targets",
                      node_id, speculative_commits.len(), view.params().n());
@@ -160,24 +162,25 @@ impl<D, ST, LP> AccessoryConsensus<D, ST, LP> for ReplicaAccessory<D, ST, LP>
 
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
-                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {}
+                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {}
 
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
-                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {}
+                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {}
 }
 
-impl<D, ST, LP> ReplicaAccessory<D, ST, LP>
+impl<D, ST, LP, RP> ReplicaAccessory<D, ST, LP, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static {
+          LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static {
     pub fn new() -> Self {
         Self {
             speculative_commits: Arc::new(Mutex::new(BTreeMap::new())),
         }
     }
 
-    fn take_speculative_commits(&self) -> BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP>>> {
+    fn take_speculative_commits(&self) -> BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP, RP>>> {
         let mut map = self.speculative_commits.lock().unwrap();
         std::mem::replace(&mut *map, BTreeMap::new())
     }
@@ -185,8 +188,8 @@ impl<D, ST, LP> ReplicaAccessory<D, ST, LP>
 
 
 #[inline]
-fn valid_spec_commits<D, ST, LP>(
-    speculative_commits: &BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP>>>,
+fn valid_spec_commits<D, ST, LP, RP>(
+    speculative_commits: &BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP, RP>>>,
     node_id: NodeId,
     seq_no: SeqNo,
     view: &ViewInfo,
@@ -194,7 +197,8 @@ fn valid_spec_commits<D, ST, LP>(
     where
         D: ApplicationData + 'static,
         ST: StateTransferMessage + 'static,
-        LP: LogTransferMessage + 'static
+        LP: LogTransferMessage + 'static,
+        RP: ReconfigurationProtocolMessage + 'static
 {
     let len = speculative_commits.len();
 
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index d75ec271..96ec4fa0 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -14,7 +14,7 @@ use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog};
-use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
 use atlas_execution::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_duration;
@@ -104,9 +104,10 @@ pub struct MessageQueue<O> {
 }
 
 /// The information needed to make a decision on a batch of requests.
-pub struct ConsensusDecision<D, ST, LP, PL> where D: ApplicationData + 'static,
-                                                  ST: StateTransferMessage + 'static,
-                                                  LP: LogTransferMessage + 'static {
+pub struct ConsensusDecision<D, ST, LP, PL, RP> where D: ApplicationData + 'static,
+                                                      ST: StateTransferMessage + 'static,
+                                                      LP: LogTransferMessage + 'static,
+                                                      RP: ReconfigurationProtocolMessage + 'static {
     node_id: NodeId,
     /// The sequence number of this consensus decision
     seq: SeqNo,
@@ -119,7 +120,7 @@ pub struct ConsensusDecision<D, ST, LP, PL> where D: ApplicationData + 'static,
     /// Persistent log reference
     persistent_log: PL,
     /// Accessory to the base consensus state machine
-    accessory: ConsensusDecisionAccessory<D, ST, LP>,
+    accessory: ConsensusDecisionAccessory<D, ST, LP, RP>,
     // Metrics about the consensus instance
     consensus_metrics: ConsensusMetrics,
     //TODO: Store things directly into the persistent log as well as delete them when
@@ -175,10 +176,11 @@ impl<O> MessageQueue<O> {
     }
 }
 
-impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
+impl<D, ST, LP, PL, RP> ConsensusDecision<D, ST, LP, PL, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static {
+          LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static {
     pub fn init_decision(node_id: NodeId, seq_no: SeqNo, view: &ViewInfo, persistent_log: PL) -> Self {
         Self {
             node_id,
@@ -266,12 +268,12 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
     pub fn process_message<NT>(&mut self,
                                header: Header,
                                message: ConsensusMessage<D::Request>,
-                               synchronizer: &Synchronizer<D>,
+                               synchronizer: &Synchronizer<D, RP>,
                                timeouts: &Timeouts,
                                log: &mut Log<D, PL>,
                                node: &Arc<NT>) -> Result<DecisionStatus>
-        where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         let view = synchronizer.view();
 
         return match self.phase {
@@ -338,8 +340,6 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
 
                 let pre_prepare_received_time = Utc::now();
 
-                let message = PBFTMessage::Consensus(message);
-
                 let stored_msg = Arc::new(ReadOnly::new(StoredMessage::new(header, message)));
 
                 let mut digests = request_batch_received(
@@ -452,7 +452,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
                     self.consensus_metrics.first_prepare_recvd();
                 }
 
-                let stored_msg = Arc::new(ReadOnly::new(StoredMessage::new(header, PBFTMessage::Consensus(message))));
+                let stored_msg = Arc::new(ReadOnly::new(StoredMessage::new(header, message)));
 
                 self.message_log.process_message(stored_msg.clone())?;
 
@@ -526,8 +526,7 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
                     self.consensus_metrics.first_commit_recvd();
                 }
 
-                let stored_msg = Arc::new(ReadOnly::new(
-                    StoredMessage::new(header, PBFTMessage::Consensus(message))));
+                let stored_msg = Arc::new(ReadOnly::new(StoredMessage::new(header,message)));
 
                 self.message_log.process_message(stored_msg.clone())?;
 
@@ -598,27 +597,32 @@ impl<D, ST, LP, PL> ConsensusDecision<D, ST, LP, PL>
     }
 }
 
-impl<D: ApplicationData + 'static, ST: StateTransferMessage + 'static, LP: LogTransferMessage + 'static, PL> Orderable for ConsensusDecision<D, ST, LP, PL> {
+impl<D, ST, LP, PL, RP> Orderable for ConsensusDecision<D, ST, LP, PL, RP>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static {
     fn sequence_number(&self) -> SeqNo {
         self.seq
     }
 }
 
 #[inline]
-fn request_batch_received<D>(
+fn request_batch_received<D, RP>(
     pre_prepare: &StoredConsensusMessage<D::Request>,
     timeouts: &Timeouts,
-    synchronizer: &Synchronizer<D>,
+    synchronizer: &Synchronizer<D, RP>,
     log: &DecidingLog<D::Request>,
 ) -> Vec<ClientRqInfo>
     where
-        D: ApplicationData + 'static
+        D: ApplicationData + 'static,
+        RP: ReconfigurationProtocolMessage + 'static
 {
     let start = Instant::now();
 
     let mut batch_guard = log.batch_meta().lock().unwrap();
 
-    batch_guard.batch_size += match pre_prepare.message().consensus().kind() {
+    batch_guard.batch_size += match pre_prepare.message().kind() {
         ConsensusMessageKind::PrePrepare(req) => {
             req.len()
         }
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 0ee037d8..1cc5ae04 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -15,7 +15,7 @@ use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage, SystemMessage};
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
 use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
@@ -184,10 +184,11 @@ pub struct Signals {
 
 /// The consensus handler. Responsible for multiplexing consensus instances and keeping track
 /// of missing messages
-pub struct Consensus<D, ST, LP, PL>
+pub struct Consensus<D, ST, LP, PL, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static,
           PL: Clone {
     node_id: NodeId,
     /// The handle to the executor of the function
@@ -203,7 +204,7 @@ pub struct Consensus<D, ST, LP, PL>
     /// The consensus instances that are currently being processed
     /// A given consensus instance n will only be finished when all consensus instances
     /// j, where j < n have already been processed, in order to maintain total ordering
-    decisions: VecDeque<ConsensusDecision<D, ST, LP, PL>>,
+    decisions: VecDeque<ConsensusDecision<D, ST, LP, PL, RP>>,
     /// The queue for messages that sit outside the range seq_no + watermark
     /// These messages cannot currently be processed since they sit outside the allowed
     /// zone but they will be processed once the seq no moves forward enough to include them
@@ -222,10 +223,11 @@ pub struct Consensus<D, ST, LP, PL>
     persistent_log: PL,
 }
 
-impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
-                                                   ST: StateTransferMessage + 'static,
-                                                   LP: LogTransferMessage + 'static,
-                                                   PL: Clone {
+impl<D, ST, LP, PL, RP> Consensus<D, ST, LP, PL, RP> where D: ApplicationData + 'static,
+                                                           ST: StateTransferMessage + 'static,
+                                                           LP: LogTransferMessage + 'static,
+                                                           RP: ReconfigurationProtocolMessage + 'static,
+                                                           PL: Clone {
     pub fn new_replica(node_id: NodeId, view: &ViewInfo, executor_handle: ExecutorHandle<D>, seq_no: SeqNo,
                        watermark: u32, consensus_guard: Arc<ProposerConsensusGuard>, timeouts: Timeouts,
                        persistent_log: PL) -> Self {
@@ -364,14 +366,14 @@ impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
     }
 
     pub fn process_message<NT>(&mut self,
-                                   header: Header,
-                                   message: ConsensusMessage<D::Request>,
-                                   synchronizer: &Synchronizer<D>,
-                                   timeouts: &Timeouts,
-                                   log: &mut Log<D, PL>,
-                                   node: &Arc<NT>) -> Result<ConsensusStatus>
-        where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D>> {
+                               header: Header,
+                               message: ConsensusMessage<D::Request>,
+                               synchronizer: &Synchronizer<D, RP>,
+                               timeouts: &Timeouts,
+                               log: &mut Log<D, PL>,
+                               node: &Arc<NT>) -> Result<ConsensusStatus>
+        where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>, {
         let message_seq = message.sequence_number();
 
         let view_seq = message.view();
@@ -483,7 +485,7 @@ impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
     /// Advance to the next instance of the consensus
     /// This will also create the necessary new decision to keep the pending decisions
     /// equal to the water mark
-    pub fn next_instance(&mut self, view: &ViewInfo) -> ConsensusDecision<D, ST, LP, PL> {
+    pub fn next_instance(&mut self, view: &ViewInfo) -> ConsensusDecision<D, ST, LP, PL, RP> {
         let decision = self.decisions.pop_front().unwrap();
 
         self.seq_no = self.seq_no.next();
@@ -550,7 +552,7 @@ impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
             }
 
             for pre_prepare in proof.pre_prepares() {
-                let x: &ConsensusMessage<D::Request> = pre_prepare.message().consensus();
+                let x: &ConsensusMessage<D::Request> = pre_prepare.message();
 
                 match x.kind() {
                     ConsensusMessageKind::PrePrepare(pre_prepare_reqs) => {
@@ -700,7 +702,8 @@ impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
                               view: &ViewInfo,
                               proof: Proof<D::Request>,
                               log: &mut Log<D, PL>) -> Result<ProtocolConsensusDecision<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where
+            PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
 
         // If this is successful, it means that we are all caught up and can now start executing the
         // batch
@@ -718,9 +721,9 @@ impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
         &self,
         requests: Vec<StoredRequestMessage<D::Request>>,
         synchronizer: &K,
-    ) -> SysMsg<D, ST, LP>
+    ) -> SysMsg<D, ST, LP, RP>
         where
-            K: AbstractSynchronizer<D>,
+            K: AbstractSynchronizer<D, RP>,
     {
         SystemMessage::from_protocol_message(PBFTMessage::Consensus(ConsensusMessage::new(
             self.sequence_number(),
@@ -801,12 +804,13 @@ impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
     pub fn finalize_view_change<NT>(
         &mut self,
         (header, message): (Header, ConsensusMessage<D::Request>),
-        synchronizer: &Synchronizer<D>,
+        synchronizer: &Synchronizer<D, RP>,
         timeouts: &Timeouts,
         log: &mut Log<D, PL>,
         node: &Arc<NT>,
-    ) where NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
-            PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    ) where
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+        PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         let view = synchronizer.view();
         //Prepare the algorithm as we are already entering this phase
 
@@ -848,7 +852,7 @@ impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
     }
 
     /// Enqueue a decision onto our overlapping decision log
-    fn enqueue_decision(&mut self, decision: ConsensusDecision<D, ST, LP, PL>) {
+    fn enqueue_decision(&mut self, decision: ConsensusDecision<D, ST, LP, PL, RP>) {
         self.signalled.push_signalled(decision.sequence_number());
 
         self.decisions.push_back(decision);
@@ -870,10 +874,11 @@ impl<D, ST, LP, PL> Consensus<D, ST, LP, PL> where D: ApplicationData + 'static,
     }
 }
 
-impl<D, ST, LP, PL> Orderable for Consensus<D, ST, LP, PL>
+impl<D, ST, LP, PL, RP> Orderable for Consensus<D, ST, LP, PL, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static,
           PL: Clone {
     fn sequence_number(&self) -> SeqNo {
         self.seq_no
diff --git a/febft-pbft-consensus/src/bft/message/mod.rs b/febft-pbft-consensus/src/bft/message/mod.rs
index b31ab93e..64a0d704 100644
--- a/febft-pbft-consensus/src/bft/message/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/mod.rs
@@ -18,6 +18,7 @@ use futures::io::{
 };
 use atlas_common::crypto::hash::{Context, Digest};
 use atlas_common::crypto::signature::{KeyPair, PublicKey, Signature};
+use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, NetworkMessage, NetworkMessageKind, PingMessage, StoredMessage};
 use atlas_execution::serialize::ApplicationData;
@@ -33,16 +34,16 @@ pub mod serialize;
 /// PBFT protocol messages
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub enum PBFTMessage<R> {
+pub enum PBFTMessage<R, JC> {
     /// Consensus message
     Consensus(ConsensusMessage<R>),
     /// View change messages
-    ViewChange(ViewChangeMessage<R>),
+    ViewChange(ViewChangeMessage<R, JC>),
     //Observer related messages
     ObserverMessage(ObserverMessage),
 }
 
-impl<R> Debug for PBFTMessage<R> {
+impl<R, JC> Debug for PBFTMessage<R, JC> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
             PBFTMessage::Consensus(_) => {
@@ -58,7 +59,7 @@ impl<R> Debug for PBFTMessage<R> {
     }
 }
 
-impl<R> Orderable for PBFTMessage<R> {
+impl<R, JC> Orderable for PBFTMessage<R, JC> {
     fn sequence_number(&self) -> SeqNo {
         match self {
             PBFTMessage::Consensus(consensus) => {
@@ -74,7 +75,7 @@ impl<R> Orderable for PBFTMessage<R> {
     }
 }
 
-impl<R> PBFTMessage<R> {
+impl<R, JC> PBFTMessage<R, JC> {
     pub fn consensus(&self) -> &ConsensusMessage<R> {
         match self {
             PBFTMessage::Consensus(msg) => msg,
@@ -89,14 +90,14 @@ impl<R> PBFTMessage<R> {
         }
     }
 
-    pub fn view_change(&self) -> &ViewChangeMessage<R> {
+    pub fn view_change(&self) -> &ViewChangeMessage<R, JC> {
         match self {
             PBFTMessage::ViewChange(msg) => msg,
             _ => panic!("Not a view change message"),
         }
     }
 
-    pub fn into_view_change(self) -> ViewChangeMessage<R> {
+    pub fn into_view_change(self) -> ViewChangeMessage<R, JC> {
         match self {
             PBFTMessage::ViewChange(msg) => msg,
             _ => panic!("Not a view change message"),
@@ -120,19 +121,19 @@ impl<R> PBFTMessage<R> {
 
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub struct ViewChangeMessage<O> {
+pub struct ViewChangeMessage<O, JC> {
     view: SeqNo,
-    kind: ViewChangeMessageKind<O>,
+    kind: ViewChangeMessageKind<O, JC>,
 }
 
-impl<O> Orderable for ViewChangeMessage<O> {
+impl<O, JC> Orderable for ViewChangeMessage<O, JC> {
     /// Returns the sequence number of the view this message refers to.
     fn sequence_number(&self) -> SeqNo {
         self.view
     }
 }
 
-impl<O> Debug for ViewChangeMessage<O> {
+impl<O, JC> Debug for ViewChangeMessage<O, JC> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(f, "View {:?}", self.view)?;
 
@@ -146,29 +147,32 @@ impl<O> Debug for ViewChangeMessage<O> {
             ViewChangeMessageKind::Sync(_) => {
                 write!(f, "Sync Message")
             }
+            ViewChangeMessageKind::NodeQuorumJoin(node_id, _) => {
+                write!(f, "Node quorum join {:?}", node_id)
+            }
         }
     }
 }
 
-impl<O> ViewChangeMessage<O> {
+impl<O, JC> ViewChangeMessage<O, JC> {
     /// Creates a new `ViewChangeMessage`, pertaining to the view
     /// with sequence number `view`, and of the kind `kind`.
-    pub fn new(view: SeqNo, kind: ViewChangeMessageKind<O>) -> Self {
+    pub fn new(view: SeqNo, kind: ViewChangeMessageKind<O, JC>) -> Self {
         Self { view, kind }
     }
 
     /// Returns a reference to the view change message kind.
-    pub fn kind(&self) -> &ViewChangeMessageKind<O> {
+    pub fn kind(&self) -> &ViewChangeMessageKind<O, JC> {
         &self.kind
     }
 
     /// Returns an owned view change message kind.
-    pub fn into_kind(self) -> ViewChangeMessageKind<O> {
+    pub fn into_kind(self) -> ViewChangeMessageKind<O, JC> {
         self.kind
     }
 
     /// Takes the collects embedded in this view change message, if they are available.
-    pub fn take_collects(self) -> Option<LeaderCollects<O>> {
+    pub fn take_collects(self) -> Option<LeaderCollects<O, JC>> {
         match self.kind {
             ViewChangeMessageKind::Sync(collects) => Some(collects),
             _ => {
@@ -180,10 +184,11 @@ impl<O> ViewChangeMessage<O> {
 
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub enum ViewChangeMessageKind<O> {
+pub enum ViewChangeMessageKind<O, JC> {
+    NodeQuorumJoin(NodeId, JC),
     Stop(Vec<StoredRequestMessage<O>>),
     StopData(CollectData<O>),
-    Sync(LeaderCollects<O>),
+    Sync(LeaderCollects<O, JC>),
 }
 
 /// Represents a message from the consensus sub-protocol.
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index 2d349d08..e133ead0 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -16,7 +16,8 @@ use bytes::Bytes;
 use atlas_common::error::*;
 use atlas_communication::serialize::Serializable;
 use atlas_core::persistent_log::PersistableOrderProtocol;
-use atlas_core::serialize::{OrderingProtocolMessage, StatefulOrderProtocolMessage};
+use atlas_core::reconfiguration_protocol::QuorumJoinCert;
+use atlas_core::serialize::{OrderingProtocolMessage, ReconfigurationProtocolMessage, StatefulOrderProtocolMessage};
 use atlas_execution::serialize::ApplicationData;
 
 use crate::bft::message::{ConsensusMessage, PBFTMessage};
@@ -61,11 +62,14 @@ pub fn deserialize_consensus<R, D>(r: R) -> Result<ConsensusMessage<D::Request>>
 }
 
 /// The serializable type, to be used to appease the compiler and it's requirements
-pub struct PBFTConsensus<D: ApplicationData>(PhantomData<D>);
+pub struct PBFTConsensus<D: ApplicationData, RP: ReconfigurationProtocolMessage>(PhantomData<(D, RP)>);
 
-impl<D> OrderingProtocolMessage for PBFTConsensus<D> where D: ApplicationData {
+impl<D, RP> OrderingProtocolMessage for PBFTConsensus<D, RP>
+    where D: ApplicationData,
+          RP: ReconfigurationProtocolMessage {
     type ViewInfo = ViewInfo;
-    type ProtocolMessage = PBFTMessage<D::Request>;
+    type ProtocolMessage = PBFTMessage<D::Request, QuorumJoinCert<RP>>;
+    type LoggableMessage = ConsensusMessage<D::Request>;
     type Proof = Proof<D::Request>;
     type ProofMetadata = ProofMetadata;
 
@@ -100,7 +104,8 @@ impl<D> OrderingProtocolMessage for PBFTConsensus<D> where D: ApplicationData {
     }
 }
 
-impl<D> StatefulOrderProtocolMessage for PBFTConsensus<D> where D: ApplicationData {
+impl<D, RP> StatefulOrderProtocolMessage for PBFTConsensus<D, RP>
+    where D: ApplicationData, RP: ReconfigurationProtocolMessage {
     type DecLog = DecisionLog<D::Request>;
 
     #[cfg(feature = "serialize_capnp")]
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 40ffe324..aa0912c2 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -26,11 +26,12 @@ use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::messages::Protocol;
-use atlas_core::ordering_protocol::{OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, ProtocolConsensusDecision, ProtocolMessage, SerProof, SerProofMetadata, View};
+use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, ProtocolConsensusDecision, ProtocolMessage, SerProof, SerProofMetadata, View};
 use atlas_core::ordering_protocol::stateful_order_protocol::{DecLog, StatefulOrderProtocol};
 use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol, StatefulOrderingProtocolLog};
+use atlas_core::reconfiguration_protocol::{QuorumJoinCert, ReconfigurationProtocol};
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
-use atlas_core::serialize::{LogTransferMessage, NetworkView, ServiceMsg, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, NetworkView, ReconfigurationProtocolMessage, ServiceMsg, StateTransferMessage};
 use atlas_core::state_transfer::{Checkpoint};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_metrics::metrics::metric_duration;
@@ -55,9 +56,9 @@ pub mod observer;
 pub mod metric;
 
 // The types responsible for this protocol
-pub type PBFT<D, ST, LP> = ServiceMsg<D, PBFTConsensus<D>, ST, LP>;
+pub type PBFT<D, ST, LP, RP> = ServiceMsg<D, PBFTConsensus<D, RP>, ST, LP>;
 // The message type for this consensus protocol
-pub type SysMsg<D, ST, LP> = <PBFT<D, ST, LP> as Serializable>::Message;
+pub type SysMsg<D, ST, LP, RP> = <PBFT<D, ST, LP, RP> as Serializable>::Message;
 
 #[derive(Clone, PartialEq, Eq, Debug)]
 /// Which phase of the consensus algorithm are we currently executing
@@ -78,25 +79,24 @@ pub enum SyncPhaseRes<O> {
 }
 
 /// a PBFT based ordering protocol
-pub struct PBFTOrderProtocol<D, ST, LP, NT, PL>
+pub struct PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
     where
         D: ApplicationData + 'static,
         ST: StateTransferMessage + 'static,
         LP: LogTransferMessage + 'static,
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
+        RP: ReconfigurationProtocolMessage + 'static,
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
         PL: Clone {
     // What phase of the consensus algorithm are we currently executing
     phase: ConsensusPhase,
-
     /// The consensus state machine
-    consensus: Consensus<D, ST, LP, PL>,
+    consensus: Consensus<D, ST, LP, PL, RP>,
     /// The synchronizer state machine
-    synchronizer: Arc<Synchronizer<D>>,
+    synchronizer: Arc<Synchronizer<D, RP>>,
     /// The request pre processor
     pre_processor: RequestPreProcessor<D::Request>,
     // A reference to the timeouts layer
     timeouts: Timeouts,
-
     //The proposer guard
     consensus_guard: Arc<ProposerConsensusGuard>,
     // Check if unordered requests can be proposed.
@@ -107,33 +107,35 @@ pub struct PBFTOrderProtocol<D, ST, LP, NT, PL>
     // Require any synchronization
     message_log: Log<D, PL>,
     // The proposer of this replica
-    proposer: Arc<Proposer<D, NT>>,
+    proposer: Arc<Proposer<D, NT, RP>>,
     // The networking layer for a Node in the network (either Client or Replica)
     node: Arc<NT>,
 
     executor: ExecutorHandle<D>,
 }
 
-impl<D, ST, LP, NT, PL> Orderable for PBFTOrderProtocol<D, ST, LP, NT, PL> where D: 'static + ApplicationData,
-                                                                                 NT: 'static + ProtocolNetworkNode<PBFT<D, ST, LP>>,
-                                                                                 ST: 'static + StateTransferMessage,
-                                                                                 LP: 'static + LogTransferMessage,
-                                                                                 PL: Clone {
+impl<D, ST, LP, NT, PL, RP> Orderable for PBFTOrderProtocol<D, ST, LP, NT, PL, RP> where D: 'static + ApplicationData,
+                                                                                         NT: 'static + ProtocolNetworkNode<PBFT<D, ST, LP, RP>>,
+                                                                                         ST: 'static + StateTransferMessage,
+                                                                                         LP: 'static + LogTransferMessage,
+                                                                                         RP: 'static + ReconfigurationProtocolMessage,
+                                                                                         PL: Clone {
     fn sequence_number(&self) -> SeqNo {
         self.consensus.sequence_number()
     }
 }
 
-impl<D, ST, LP, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, LP, NT, PL>
+impl<D, ST, LP, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+          RP: ReconfigurationProtocolMessage + 'static,
           PL: Clone {
-    type Serialization = PBFTConsensus<D>;
-    type Config = PBFTConfig<D, ST>;
+    type Serialization = PBFTConsensus<D, RP>;
+    type Config = PBFTConfig<D, ST, RP>;
 
-    fn initialize(config: PBFTConfig<D, ST>, args: OrderingProtocolArgs<D, NT, PL>) -> Result<Self> where
+    fn initialize(config: PBFTConfig<D, ST, RP>, args: OrderingProtocolArgs<D, NT, PL>) -> Result<Self> where
         Self: Sized,
     {
         Self::initialize_protocol(config, args, None)
@@ -143,8 +145,8 @@ impl<D, ST, LP, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST,
         self.synchronizer.view()
     }
 
-    fn handle_off_ctx_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>)
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    fn handle_off_ctx_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request, QuorumJoinCert<RP>>>>)
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         let (header, message) = message.into_inner();
 
         match message.into_inner() {
@@ -162,8 +164,8 @@ impl<D, ST, LP, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST,
         }
     }
 
-    fn poll(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    fn poll(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request, QuorumJoinCert<RP>>, D::Request>
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         match self.phase {
             ConsensusPhase::NormalPhase => {
                 self.poll_normal_phase()
@@ -174,8 +176,8 @@ impl<D, ST, LP, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST,
         }
     }
 
-    fn process_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    fn process_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request, QuorumJoinCert<RP>>>>) -> Result<OrderProtocolExecResult<D::Request>>
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         match self.phase {
             ConsensusPhase::NormalPhase => {
                 self.update_normal_phase(message)
@@ -187,7 +189,7 @@ impl<D, ST, LP, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST,
     }
 
     fn handle_timeout(&mut self, timeout: Vec<RqTimeout>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         if self.consensus.is_catching_up() {
             warn!("{:?} // Ignoring timeouts while catching up", self.node.id());
 
@@ -263,12 +265,14 @@ impl<D, ST, LP, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST,
     }
 }
 
-impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: ApplicationData + 'static,
-                                                                   ST: StateTransferMessage + 'static,
-                                                                   LP: LogTransferMessage + 'static,
-                                                                   NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
-                                                                   PL: Clone {
-    fn initialize_protocol(config: PBFTConfig<D, ST>, args: OrderingProtocolArgs<D, NT, PL>,
+impl<D, ST, LP, NT, PL, RP> PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+          PL: Clone {
+    fn initialize_protocol(config: PBFTConfig<D, ST, RP>, args: OrderingProtocolArgs<D, NT, PL>,
                            initial_state: Option<DecisionLog<D::Request>>) -> Result<Self> {
         let PBFTConfig {
             node_id,
@@ -284,15 +288,15 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
 
         let consensus_guard = ProposerConsensusGuard::new(view.clone(), watermark);
 
-        let consensus = Consensus::<D, ST, LP, PL>::new_replica(node_id, &sync.view(), executor.clone(),
-                                                            SeqNo::ZERO, watermark, consensus_guard.clone(),
-                                                            timeouts.clone(), persistent_log.clone());
+        let consensus = Consensus::<D, ST, LP, PL, RP>::new_replica(node_id, &sync.view(), executor.clone(),
+                                                                    SeqNo::ZERO, watermark, consensus_guard.clone(),
+                                                                    timeouts.clone(), persistent_log.clone());
 
         let dec_log = initialize_decided_log::<D, PL>(node_id, persistent_log, initial_state)?;
 
         let proposer = Proposer::<D, NT, RP>::new(node.clone(), batch_input, sync.clone(), timeouts.clone(),
-                                              executor.clone(), consensus_guard.clone(),
-                                              proposer_config);
+                                                  executor.clone(), consensus_guard.clone(),
+                                                  proposer_config);
 
         let replica = Self {
             phase: ConsensusPhase::NormalPhase,
@@ -327,8 +331,8 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
         Ok(replica)
     }
 
-    fn poll_sync_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    fn poll_sync_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request, QuorumJoinCert<RP>>, D::Request>
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
 
         // retrieve a view change message to be processed
         let poll_result = self.synchronizer.poll();
@@ -357,8 +361,8 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
         }
     }
 
-    fn poll_normal_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    fn poll_normal_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request, QuorumJoinCert<RP>>, D::Request>
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         // check if we have STOP messages to be processed,
         // and update our phase when we start installing
         // the new view
@@ -410,8 +414,8 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
         }
     }
 
-    fn update_sync_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    fn update_sync_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request, QuorumJoinCert<RP>>>>) -> Result<OrderProtocolExecResult<D::Request>>
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         let (header, protocol) = message.into_inner();
 
         match protocol.into_inner() {
@@ -447,8 +451,8 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
         Ok(OrderProtocolExecResult::Success)
     }
 
-    fn update_normal_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    fn update_normal_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request, QuorumJoinCert<RP>>>>) -> Result<OrderProtocolExecResult<D::Request>>
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         let (header, protocol) = message.into_inner();
 
         match protocol.into_inner() {
@@ -491,7 +495,7 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
         header: Header,
         message: ConsensusMessage<D::Request>,
     ) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         let seq = self.consensus.sequence_number();
 
         // debug!(
@@ -558,8 +562,8 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
 
     /// Advance the sync phase of the algorithm
     fn adv_sync(&mut self, header: Header,
-                message: ViewChangeMessage<D::Request>) -> SyncPhaseRes<D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+                message: ViewChangeMessage<D::Request, QuorumJoinCert<RP>>) -> SyncPhaseRes<D::Request>
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         let status = self.synchronizer.process_message(
             header,
             message,
@@ -603,13 +607,14 @@ impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL> where D: Applicatio
     }
 }
 
-impl<D, ST, LP, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, LP, NT, PL>
+impl<D, ST, LP, NT, PL, RP> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
+          RP: ReconfigurationProtocolMessage + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
           PL: Clone {
-    type StateSerialization = PBFTConsensus<D>;
+    type StateSerialization = PBFTConsensus<D, RP>;
 
     fn initialize_with_initial_state(config: Self::Config,
                                      args: OrderingProtocolArgs<D, NT, PL>,
@@ -620,7 +625,7 @@ impl<D, ST, LP, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D
     fn install_state(&mut self,
                      view_info: View<Self::Serialization>,
                      dec_log: DecLog<Self::StateSerialization>) -> Result<Vec<D::Request>>
-        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
+        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D, RP>, PBFTConsensus<D, RP>> {
         info!("{:?} // Installing decision log with Seq No {:?} and View {:?}", self.node.id(),
                 dec_log.sequence_number(), view_info);
 
@@ -667,11 +672,12 @@ impl<D, ST, LP, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D
     }
 }
 
-impl<D, ST, LP, NT, PL> PBFTOrderProtocol<D, ST, LP, NT, PL>
+impl<D, ST, LP, NT, PL, RP> PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
+          RP: ReconfigurationProtocolMessage + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
           PL: Clone {
     pub(crate) fn switch_phase(&mut self, new_phase: ConsensusPhase) {
         info!("{:?} // Switching from phase {:?} to phase {:?}", self.node.id(), self.phase, new_phase);
@@ -731,72 +737,61 @@ const CF_PRE_PREPARES: &str = "PRE_PREPARES";
 const CF_PREPARES: &str = "PREPARES";
 const CF_COMMIT: &str = "COMMITS";
 
-impl<D, ST, LP, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus<D>> for PBFTOrderProtocol<D, ST, LP, NT, PL>
+impl<D, ST, LP, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTConsensus<D, RP>> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP>>, PL: Clone {
+          RP: ReconfigurationProtocolMessage + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>, PL: Clone {
     fn message_types() -> Vec<&'static str> {
         vec![
             CF_PRE_PREPARES,
-            CF_PREPARES, CF_COMMIT,
+            CF_PREPARES,
+            CF_COMMIT,
         ]
     }
 
-    fn get_type_for_message(msg: &ProtocolMessage<PBFTConsensus<D>>) -> Result<&'static str> {
-        match msg {
-            PBFTMessage::Consensus(msg) => {
-                match msg.kind() {
-                    ConsensusMessageKind::PrePrepare(_) => {
-                        Ok(CF_PRE_PREPARES)
-                    }
-                    ConsensusMessageKind::Prepare(_) => {
-                        Ok(CF_PREPARES)
-                    }
-                    ConsensusMessageKind::Commit(_) => {
-                        Ok(CF_COMMIT)
-                    }
-                }
+    fn get_type_for_message(msg: &LoggableMessage<PBFTConsensus<D, RP>>) -> Result<&'static str> {
+        match msg.kind() {
+            ConsensusMessageKind::PrePrepare(_) => {
+                Ok(CF_PRE_PREPARES)
             }
-            _ => {
-                Err(Error::simple_with_msg(ErrorKind::MsgLogPersistentSerialization, "Invalid message type"))
+            ConsensusMessageKind::Prepare(_) => {
+                Ok(CF_PREPARES)
+            }
+            ConsensusMessageKind::Commit(_) => {
+                Ok(CF_COMMIT)
             }
         }
     }
 
-    fn init_proof_from(metadata: SerProofMetadata<PBFTConsensus<D>>, messages: Vec<StoredMessage<ProtocolMessage<PBFTConsensus<D>>>>) -> SerProof<PBFTConsensus<D>> {
+    fn init_proof_from(metadata: SerProofMetadata<PBFTConsensus<D, RP>>, messages: Vec<StoredMessage<LoggableMessage<PBFTConsensus<D, RP>>>>) -> SerProof<PBFTConsensus<D, RP>> {
         let mut pre_prepares = Vec::with_capacity(messages.len() / 2);
         let mut prepares = Vec::with_capacity(messages.len() / 2);
         let mut commits = Vec::with_capacity(messages.len() / 2);
 
         for message in messages {
-            match message.message() {
-                PBFTMessage::Consensus(cons) => {
-                    match cons.kind() {
-                        ConsensusMessageKind::PrePrepare(_) => {
-                            pre_prepares.push(Arc::new(ReadOnly::new(message)));
-                        }
-                        ConsensusMessageKind::Prepare(_) => {
-                            prepares.push(Arc::new(ReadOnly::new(message)));
-                        }
-                        ConsensusMessageKind::Commit(_) => {
-                            commits.push(Arc::new(ReadOnly::new(message)));
-                        }
-                    }
+            match message.message().kind() {
+                ConsensusMessageKind::PrePrepare(_) => {
+                    pre_prepares.push(Arc::new(ReadOnly::new(message)));
+                }
+                ConsensusMessageKind::Prepare(_) => {
+                    prepares.push(Arc::new(ReadOnly::new(message)));
+                }
+                ConsensusMessageKind::Commit(_) => {
+                    commits.push(Arc::new(ReadOnly::new(message)));
                 }
-                PBFTMessage::ViewChange(_) => { unreachable!() }
-                PBFTMessage::ObserverMessage(_) => { unreachable!() }
             }
         }
 
         Proof::new(metadata, pre_prepares, prepares, commits)
     }
 
-    fn init_dec_log(proofs: Vec<SerProof<PBFTConsensus<D>>>) -> DecLog<PBFTConsensus<D>> {
+    fn init_dec_log(proofs: Vec<SerProof<PBFTConsensus<D, RP>>>) -> DecLog<PBFTConsensus<D, RP>> {
         DecisionLog::from_proofs(proofs)
     }
 
-    fn decompose_proof(proof: &SerProof<PBFTConsensus<D>>) -> (&SerProofMetadata<PBFTConsensus<D>>, Vec<&StoredMessage<ProtocolMessage<PBFTConsensus<D>>>>) {
+    fn decompose_proof(proof: &SerProof<PBFTConsensus<D, RP>>) -> (&SerProofMetadata<PBFTConsensus<D, RP>>, Vec<&StoredMessage<LoggableMessage<PBFTConsensus<D, RP>>>>) {
         let mut messages = Vec::new();
 
         for message in proof.pre_prepares() {
@@ -814,7 +809,7 @@ impl<D, ST, LP, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus
         (proof.metadata(), messages)
     }
 
-    fn decompose_dec_log(proofs: &DecLog<PBFTConsensus<D>>) -> Vec<&SerProof<PBFTConsensus<D>>> {
+    fn decompose_dec_log(proofs: &DecLog<PBFTConsensus<D, RP>>) -> Vec<&SerProof<PBFTConsensus<D, RP>>> {
         proofs.proofs().iter().collect()
     }
 }
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index 616e84f7..a0a3697b 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -9,7 +9,7 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::StoredMessage;
 use atlas_core::ordering_protocol::{DecisionInformation, ProtocolConsensusDecision};
 use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_core::serialize::StateTransferMessage;
+use atlas_core::serialize::{ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_execution::app::UpdateBatch;
 use atlas_execution::serialize::ApplicationData;
 
@@ -17,7 +17,7 @@ use crate::bft::message::{ConsensusMessageKind, PBFTMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::msg_log::operation_key;
 use crate::bft::msg_log::deciding_log::CompletedBatch;
-use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata};
+use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata, StoredConsensusMessage};
 use crate::bft::sync::view::ViewInfo;
 
 /// The log of decisions that have already been processed by the consensus
@@ -52,8 +52,9 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// Read the current state, if existent, from the persistent storage
     ///
     /// FIXME: The view initialization might have to be changed if we want to introduce reconfiguration
-    pub fn read_current_state(&self, n: usize, f: usize) -> Result<Option<(ViewInfo, DecisionLog<D::Request>)>>
-        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
+    pub fn read_current_state<RP>(&self, n: usize, f: usize) -> Result<Option<(ViewInfo, DecisionLog<D::Request>)>>
+        where RP: ReconfigurationProtocolMessage + 'static,
+              PL: StatefulOrderingProtocolLog<PBFTConsensus<D, RP>, PBFTConsensus<D, RP>> {
         let option = self.persistent_log.read_state(OperationMode::BlockingSync)?;
 
         if let Some((view, dec_log)) = option {
@@ -76,10 +77,12 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// We can use this method when we want to prevent a clone, as this takes
     /// just a reference.
     /// This is mostly used for pre prepares as they contain all the requests and are therefore very expensive to send
-    pub fn insert_consensus(
+    pub fn insert_consensus<RP>(
         &mut self,
-        consensus_msg: Arc<ReadOnly<StoredMessage<PBFTMessage<D::Request>>>>,
-    ) where PL: OrderingProtocolLog<PBFTConsensus<D>>,
+        consensus_msg: StoredConsensusMessage<D::Request>,
+    ) where
+        RP: ReconfigurationProtocolMessage + 'static,
+        PL: OrderingProtocolLog<PBFTConsensus<D, RP>>,
     {
         if let Err(err) = self
             .persistent_log
@@ -93,8 +96,10 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// This is done when we receive the final SYNC message from the leader
     /// which contains all of the collects
     /// If we are missing the request determined by the
-    pub fn install_proof(&mut self, seq: SeqNo, proof: Proof<D::Request>) -> Result<ProtocolConsensusDecision<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    pub fn install_proof<RP>(&mut self, seq: SeqNo, proof: Proof<D::Request>) -> Result<ProtocolConsensusDecision<D::Request>>
+        where
+            RP: ReconfigurationProtocolMessage + 'static,
+            PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         let batch_execution_info = ProtocolConsensusDecision::from(&proof);
 
         if let Some(decision) = self.decision_log().last_decision() {
@@ -118,17 +123,20 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     }
 
     /// Clear the occurrences of a seq no from the decision log
-    pub fn clear_last_occurrence(&mut self, seq: SeqNo)
+    pub fn clear_last_occurrence<RP>(&mut self, seq: SeqNo)
         where
-            PL: OrderingProtocolLog<PBFTConsensus<D>> {
+            RP: ReconfigurationProtocolMessage + 'static,
+            PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         if let Err(err) = self.persistent_log.write_invalidate(OperationMode::NonBlockingSync(None), seq) {
             error!("Failed to invalidate last occurrence {:?}", err);
         }
     }
 
     /// Update the log state, received from the CST protocol.
-    pub fn install_state(&mut self, view: ViewInfo, dec_log: DecisionLog<D::Request>)
-        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
+    pub fn install_state<RP>(&mut self, view: ViewInfo, dec_log: DecisionLog<D::Request>)
+        where
+            RP: ReconfigurationProtocolMessage + 'static,
+            PL: StatefulOrderingProtocolLog<PBFTConsensus<D, RP>, PBFTConsensus<D, RP>> {
 
         //Replace the log
         self.dec_log = dec_log.clone();
@@ -163,8 +171,9 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
 
     /// Register that all the batches for a given decision have already been received
     /// Basically persists the metadata for a given consensus num
-    pub fn all_batches_received(&mut self, metadata: ProofMetadata) where
-        PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    pub fn all_batches_received<RP>(&mut self, metadata: ProofMetadata)
+        where RP: ReconfigurationProtocolMessage + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         self.persistent_log.write_proof_metadata(OperationMode::NonBlockingSync(None),
                                                  metadata).unwrap();
     }
@@ -191,7 +200,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
 
             for message in completed_batch.pre_prepare_messages() {
                 let reqs = {
-                    if let ConsensusMessageKind::PrePrepare(reqs) = (*message.message().consensus().kind()).clone() {
+                    if let ConsensusMessageKind::PrePrepare(reqs) = (*message.message().kind()).clone() {
                         reqs
                     } else { unreachable!() }
                 };
@@ -257,7 +266,7 @@ impl<O> From<&Proof<O>> for ProtocolConsensusDecision<O> where O: Clone {
         for pre_prepare in value.pre_prepares() {
             let consensus_msg = (*pre_prepare.message()).clone();
 
-            let reqs = match consensus_msg.into_consensus().into_kind() {
+            let reqs = match consensus_msg.into_kind() {
                 ConsensusMessageKind::PrePrepare(reqs) => { reqs }
                 _ => {
                     unreachable!()
diff --git a/febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs
index df84f4a3..cb01f717 100644
--- a/febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs
@@ -4,19 +4,19 @@ use std::iter;
 use std::iter::zip;
 use std::sync::{Arc, Mutex};
 use std::time::Instant;
+
 use atlas_common::crypto::hash::{Context, Digest};
-use atlas_common::globals::ReadOnly;
 use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_communication::message::StoredMessage;
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::ordering_protocol::DecisionInformation;
 use atlas_metrics::benchmarks::BatchMeta;
 use atlas_metrics::metrics::metric_duration;
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
+
+use crate::bft::message::ConsensusMessageKind;
 use crate::bft::metric::PRE_PREPARE_LOG_ANALYSIS_ID;
-use crate::bft::msg_log::decisions::{IncompleteProof, Proof, ProofMetadata, StoredConsensusMessage, ViewDecisionPair, PrepareSet};
+use crate::bft::msg_log::decisions::{IncompleteProof, PrepareSet, Proof, ProofMetadata, StoredConsensusMessage, ViewDecisionPair};
 use crate::bft::sync::view::ViewInfo;
 
 /// A batch that has been decided by the consensus instance and is now ready to be delivered to the
@@ -226,7 +226,7 @@ impl<O> DecidingLog<O> {
 
     /// Process the message received
     pub(crate) fn process_message(&mut self, message: StoredConsensusMessage<O>) -> Result<()> {
-        match message.message().consensus().kind() {
+        match message.message().kind() {
             ConsensusMessageKind::Prepare(_) => {
                 self.duplicate_detection.insert_prepare_received(message.header().from())?;
             }
@@ -317,7 +317,7 @@ impl<O> OnGoingDecision<O> {
 
     /// Insert a consensus message into this on going decision
     fn insert_message(&mut self, message: StoredConsensusMessage<O>) {
-        match message.message().consensus().kind() {
+        match message.message().kind() {
             ConsensusMessageKind::Prepare(_) => {
                 self.prepare_messages.push(message);
             }
@@ -332,7 +332,7 @@ impl<O> OnGoingDecision<O> {
 
     /// Insert a message from the stored message into this on going decision
     pub fn insert_persisted_msg(&mut self, message: StoredConsensusMessage<O>) -> Result<()> {
-        match message.message().consensus().kind() {
+        match message.message().kind() {
             ConsensusMessageKind::PrePrepare(_) => {
                 let index = pre_prepare_index_from_digest_opt(&self.pre_prepare_digests, message.header().digest())?;
 
@@ -354,15 +354,15 @@ impl<O> OnGoingDecision<O> {
             let mut buf = Vec::new();
 
             for stored in self.prepare_messages.iter().rev() {
-                match stored.message().consensus().sequence_number().cmp(&in_exec) {
+                match stored.message().sequence_number().cmp(&in_exec) {
                     Ordering::Equal => {
-                        let digest = match stored.message().consensus().kind() {
+                        let digest = match stored.message().kind() {
                             ConsensusMessageKind::Prepare(d) => d.clone(),
                             _ => unreachable!(),
                         };
 
                         buf.push(ViewDecisionPair(
-                            stored.message().consensus().view(),
+                            stored.message().view(),
                             digest,
                         ));
                     }
@@ -383,21 +383,21 @@ impl<O> OnGoingDecision<O> {
             let mut count = 0;
 
             for stored in self.prepare_messages.iter().rev() {
-                match stored.message().consensus().sequence_number().cmp(&in_exec) {
+                match stored.message().sequence_number().cmp(&in_exec) {
                     Ordering::Equal => {
                         match last_view {
                             None => (),
-                            Some(v) if stored.message().consensus().view() == v => (),
+                            Some(v) if stored.message().view() == v => (),
                             _ => count = 0,
                         }
-                        last_view = Some(stored.message().consensus().view());
+                        last_view = Some(stored.message().view());
                         count += 1;
                         if count == quorum {
-                            let digest = match stored.message().consensus().kind() {
+                            let digest = match stored.message().kind() {
                                 ConsensusMessageKind::Prepare(d) => d.clone(),
                                 _ => unreachable!(),
                             };
-                            break 'outer Some(ViewDecisionPair(stored.message().consensus().view(), digest));
+                            break 'outer Some(ViewDecisionPair(stored.message().view(), digest));
                         }
                     }
                     Ordering::Less => break,
diff --git a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
index ecdb243c..2a8af4d6 100644
--- a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
@@ -1,21 +1,21 @@
-use std::cmp::Ordering;
 use std::fmt::{Debug, Formatter};
 use std::ops::Deref;
 use std::sync::Arc;
-use atlas_common::error::*;
 
 #[cfg(feature = "serialize_serde")]
 use serde::{Deserialize, Serialize};
+
 use atlas_common::crypto::hash::Digest;
+use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::StoredMessage;
 use atlas_core::serialize::{OrderProtocolLog, OrderProtocolProof};
-use atlas_execution::serialize::ApplicationData;
+
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::msg_log::deciding_log::CompletedBatch;
 
-pub type StoredConsensusMessage<O> = Arc<ReadOnly<StoredMessage<PBFTMessage<O>>>>;
+pub type StoredConsensusMessage<O> = Arc<ReadOnly<StoredMessage<ConsensusMessage<O>>>>;
 
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
@@ -352,7 +352,7 @@ impl<O> DecisionLog<O> {
             if proof.seq_no <= seq_no {
                 for pre_prepare in &proof.pre_prepares {
                     //Mark the requests contained in this message for removal
-                    decided_request_count += match pre_prepare.message().consensus().kind() {
+                    decided_request_count += match pre_prepare.message().kind() {
                         ConsensusMessageKind::PrePrepare(messages) => messages.len(),
                         _ => 0,
                     };
diff --git a/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs
index 46dc2308..ade733d6 100644
--- a/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/follower_proposer/mod.rs
@@ -5,7 +5,7 @@ use atlas_common::channel;
 use atlas_common::channel::{ChannelSyncRx, ChannelSyncTx};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::StoredRequestMessage;
-use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
 
@@ -14,11 +14,12 @@ use crate::bft::PBFT;
 pub type BatchType<D: ApplicationData> = Vec<StoredRequestMessage<D::Request>>;
 
 ///TODO:
-pub struct FollowerProposer<D, ST, LP, NT>
+pub struct FollowerProposer<D, ST, LP, NT, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+          RP: ReconfigurationProtocolMessage + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
     batch_channel: (ChannelSyncTx<BatchType<D>>, ChannelSyncRx<BatchType<D>>),
     //For request execution
     executor_handle: ExecutorHandle<D>,
@@ -31,7 +32,7 @@ pub struct FollowerProposer<D, ST, LP, NT>
     target_global_batch_size: usize,
     //Time limit for generating a batch with target_global_batch_size size
     global_batch_time_limit: u128,
-    _phantom: PhantomData<(ST, LP)>
+    _phantom: PhantomData<(ST, LP)>,
 }
 
 
@@ -39,11 +40,12 @@ pub struct FollowerProposer<D, ST, LP, NT>
 const BATCH_CHANNEL_SIZE: usize = 1024;
 
 
-impl<D, ST, LP, NT> FollowerProposer<D, ST, LP, NT>
+impl<D, ST, LP, NT, RP> FollowerProposer<D, ST, LP, NT, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+          RP: ReconfigurationProtocolMessage + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
     pub fn new(
         node: Arc<NT>,
         executor: ExecutorHandle<D>,
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index 4b48a6cb..eb2dc609 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -10,12 +10,11 @@ use atlas_common::channel::TryRecvError;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::threadpool;
-use atlas_communication::message::NetworkMessageKind;
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
 use atlas_core::reconfiguration_protocol::ReconfigurationProtocol;
 use atlas_core::request_pre_processing::{BatchOutput, PreProcessorOutputMessage};
-use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
 use atlas_execution::app::UnorderedBatch;
 use atlas_execution::ExecutorHandle;
@@ -31,20 +30,21 @@ use crate::bft::sync::view::{is_request_in_hash_space, ViewInfo};
 
 use super::sync::{AbstractSynchronizer, Synchronizer};
 
-pub mod follower_proposer;
+//pub mod follower_proposer;
 
 pub type BatchType<R> = Vec<StoredRequestMessage<R>>;
 
 ///Handles taking requests from the client pools and storing the requests in the log,
 ///as well as creating new batches and delivering them to the batch_channel
 ///Another thread will then take from this channel and propose the requests
-pub struct Proposer<D, NT>
-    where D: ApplicationData + 'static {
+pub struct Proposer<D, NT, RP>
+    where D: ApplicationData + 'static,
+          RP: ReconfigurationProtocolMessage + 'static {
     /// Channel for the reception of batches from the pre processing module
     batch_reception: BatchOutput<D::Request>,
     /// Network Node
     node_ref: Arc<NT>,
-    synchronizer: Arc<Synchronizer<D>>,
+    synchronizer: Arc<Synchronizer<D, RP>>,
     timeouts: Timeouts,
     consensus_guard: Arc<ProposerConsensusGuard>,
     // Should we shut down?
@@ -78,18 +78,18 @@ impl<D> ProposeBuilder<D> where D: ApplicationData {
 ///The size of the batch channel
 const BATCH_CHANNEL_SIZE: usize = 128;
 
-impl<D, NT> Proposer<D, NT>
-    where D: ApplicationData + 'static, {
+impl<D, NT, RP> Proposer<D, NT, RP>
+    where D: ApplicationData + 'static,
+          RP: ReconfigurationProtocolMessage + 'static {
     pub fn new(
         node: Arc<NT>,
         batch_input: BatchOutput<D::Request>,
-        sync: Arc<Synchronizer<D>>,
+        sync: Arc<Synchronizer<D, RP>>,
         timeouts: Timeouts,
         executor_handle: ExecutorHandle<D>,
         consensus_guard: Arc<ProposerConsensusGuard>,
         proposer_config: ProposerConfig,
     ) -> Arc<Self> {
-
         let ProposerConfig {
             target_batch_size, max_batch_size, batch_timeout
         } = proposer_config;
@@ -112,7 +112,7 @@ impl<D, NT> Proposer<D, NT>
     pub fn start<ST, LP>(self: Arc<Self>) -> JoinHandle<()>
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static {
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static {
         std::thread::Builder::new()
             .name(format!("Proposer thread"))
             .spawn(move || {
@@ -252,7 +252,7 @@ impl<D, NT> Proposer<D, NT>
         propose: &mut ProposeBuilder<D>,
     ) -> bool where ST: StateTransferMessage + 'static,
                     LP: LogTransferMessage + 'static,
-                    NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+                    NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         if !propose.currently_accumulated.is_empty() {
             let current_batch_size = propose.currently_accumulated.len();
 
@@ -323,7 +323,7 @@ impl<D, NT> Proposer<D, NT>
                                propose: &mut ProposeBuilder<D>, ) -> bool
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
 
         //Now let's deal with ordered requests
         if is_leader {
@@ -383,7 +383,7 @@ impl<D, NT> Proposer<D, NT>
     ) where
         ST: StateTransferMessage + 'static,
         LP: LogTransferMessage + 'static,
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         let has_pending_messages = self.consensus_guard.has_pending_view_change_reqs();
 
         let is_view_change_empty = {
diff --git a/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
index 2321b3d4..49b535a0 100644
--- a/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
@@ -24,7 +24,7 @@ impl<D: ApplicationData + 'static> FollowerSynchronizer<D> {
         pre_prepare: &StoredConsensusMessage<D::Request>,
     ) -> Vec<ClientRqInfo> {
 
-        let requests = match pre_prepare.message().consensus().kind() {
+        let requests = match pre_prepare.message().kind() {
             ConsensusMessageKind::PrePrepare(req) => {req},
             _ => {panic!()}
         };
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 0e475eb2..f7ac36d2 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -25,8 +25,9 @@ use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
+use atlas_core::reconfiguration_protocol::QuorumJoinCert;
 use atlas_core::request_pre_processing::RequestPreProcessor;
-use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_execution::serialize::ApplicationData;
 
@@ -49,12 +50,12 @@ pub mod view;
 /// The code provided in the first argument gets executed
 /// The first T is the type of message that we should expect to be returned from the queue
 macro_rules! extract_msg {
-    ($t:ty => $g:expr, $q:expr) => {
-        extract_msg!($t => {}, $g, $q)
+    ($t:ty, $t2:ty => $g:expr, $q:expr) => {
+        extract_msg!($t, $t2 => {}, $g, $q)
     };
 
-    ($t:ty => $opt:block, $g:expr, $q:expr) => {
-        if let Some(stored) = tbo_pop_message::<StoredMessage<ViewChangeMessage<$t>>>($q) {
+    ($t:ty, $t2:ty => $opt:block, $g:expr, $q:expr) => {
+        if let Some(stored) = tbo_pop_message::<StoredMessage<ViewChangeMessage<$t, $t2>>>($q) {
             $opt
             let (header, message) = stored.into_inner();
             SynchronizerPollStatus::NextMessage(header, message)
@@ -111,21 +112,21 @@ macro_rules! finalize_view_change {
 /// of the view change protocol, as well as a value to be proposed in the `SYNC` message.
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub struct LeaderCollects<O> {
+pub struct LeaderCollects<O, JC> {
     //The pre prepare message, created and signed by the leader to be executed when the view change is
     // Done
     proposed: FwdConsensusMessage<O>,
     // The collect messages the leader has received.
-    collects: Vec<StoredMessage<ViewChangeMessage<O>>>,
+    collects: Vec<StoredMessage<ViewChangeMessage<O, JC>>>,
 }
 
-impl<O> LeaderCollects<O> {
+impl<O, JC> LeaderCollects<O, JC> {
     /// Gives up ownership of the inner values of this `LeaderCollects`.
     pub fn into_inner(
         self,
     ) -> (
         FwdConsensusMessage<O>,
-        Vec<StoredMessage<ViewChangeMessage<O>>>,
+        Vec<StoredMessage<ViewChangeMessage<O, JC>>>,
     ) {
         (self.proposed, self.collects)
     }
@@ -168,7 +169,7 @@ impl Sound {
 }
 
 /// Represents a queue of view change messages that arrive out of context into a node.
-pub struct TboQueue<O> {
+pub struct TboQueue<O, JC> {
     // the current view
     view: ViewInfo,
     // Stores the previous view, for useful information when changing views
@@ -177,14 +178,14 @@ pub struct TboQueue<O> {
     // fetching them from the network
     get_queue: bool,
     // stores all STOP messages for the next view
-    stop: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O>>>>,
+    stop: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O, JC>>>>,
     // stores all STOP-DATA messages for the next view
-    stop_data: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O>>>>,
+    stop_data: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O, JC>>>>,
     // stores all SYNC messages for the next view
-    sync: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O>>>>,
+    sync: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O, JC>>>>,
 }
 
-impl<O> TboQueue<O> {
+impl<O, JC> TboQueue<O, JC> {
     pub(crate) fn new(view: ViewInfo) -> Self {
         Self {
             view,
@@ -237,7 +238,7 @@ impl<O> TboQueue<O> {
 
     /// Queues a view change message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    pub fn queue(&mut self, h: Header, m: ViewChangeMessage<O>) {
+    pub fn queue(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
         match m.kind() {
             ViewChangeMessageKind::Stop(_) => self.queue_stop(h, m),
             ViewChangeMessageKind::StopData(_) => self.queue_stop_data(h, m),
@@ -256,7 +257,7 @@ impl<O> TboQueue<O> {
 
     /// Queues a `STOP` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    fn queue_stop(&mut self, h: Header, m: ViewChangeMessage<O>) {
+    fn queue_stop(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
         // NOTE: we use next() because we want to retrieve messages
         // for v+1, as we haven't started installing the new view yet
         let seq = self.view.sequence_number().next();
@@ -265,14 +266,14 @@ impl<O> TboQueue<O> {
 
     /// Queues a `STOP-DATA` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    fn queue_stop_data(&mut self, h: Header, m: ViewChangeMessage<O>) {
+    fn queue_stop_data(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
         let seq = self.view.sequence_number();
         tbo_queue_message(seq, &mut self.stop_data, StoredMessage::new(h, m))
     }
 
     /// Queues a `SYNC` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    fn queue_sync(&mut self, h: Header, m: ViewChangeMessage<O>) {
+    fn queue_sync(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
         let seq = self.view.sequence_number();
         tbo_queue_message(seq, &mut self.sync, StoredMessage::new(h, m))
     }
@@ -336,21 +337,21 @@ pub enum SynchronizerStatus<O> {
 
 /// Represents the status of calling `poll()` on a `Synchronizer`.
 #[derive(Clone)]
-pub enum SynchronizerPollStatus<O> {
+pub enum SynchronizerPollStatus<O, JC> {
     /// The `Replica` associated with this `Synchronizer` should
     /// poll its network channel for more messages, as it has no messages
     /// That can be processed in cache
     Recv,
     /// A new view change message is available to be processed, retrieved from the
     /// Ordered queue
-    NextMessage(Header, ViewChangeMessage<O>),
+    NextMessage(Header, ViewChangeMessage<O, JC>),
     /// We need to resume the view change protocol, after
     /// running the CST protocol.
     ResumeViewChange,
 }
 
 ///A trait describing some of the necessary methods for the synchronizer
-pub trait AbstractSynchronizer<D: ApplicationData + 'static> {
+pub trait AbstractSynchronizer<D, RP> where D: ApplicationData + 'static, RP: ReconfigurationProtocolMessage + 'static {
     /// Returns information regarding the current view, such as
     /// the number of faulty replicas the system can tolerate.
     fn view(&self) -> ViewInfo;
@@ -359,23 +360,23 @@ pub trait AbstractSynchronizer<D: ApplicationData + 'static> {
     /// running the view change protocol.
     fn install_view(&self, view: ViewInfo);
 
-    fn queue(&self, header: Header, message: ViewChangeMessage<D::Request>);
+    fn queue(&self, header: Header, message: ViewChangeMessage<D::Request, QuorumJoinCert<RP>>);
 }
 
-type CollectsType<D: ApplicationData> = IntMap<StoredMessage<ViewChangeMessage<D::Request>>>;
+type CollectsType<D: ApplicationData, JC> = IntMap<StoredMessage<ViewChangeMessage<D::Request, JC>>>;
 
 ///The synchronizer for the SMR protocol
 /// This part of the protocol is responsible for handling the changing of views and
 /// for keeping track of any timed out client requests
-pub struct Synchronizer<D: ApplicationData> {
+pub struct Synchronizer<D: ApplicationData, RP: ReconfigurationProtocolMessage> {
     phase: Cell<ProtoPhase>,
     //Tbo queue, keeps track of the current view and keeps messages arriving in order
-    tbo: Mutex<TboQueue<D::Request>>,
+    tbo: Mutex<TboQueue<D::Request, QuorumJoinCert<RP>>>,
     //Stores currently received requests from other nodes
     stopped: RefCell<IntMap<Vec<StoredRequestMessage<D::Request>>>>,
     //TODO: This does not require a Mutex I believe since it's only accessed when
     // Processing messages (which is always done in the replica thread)
-    collects: Mutex<CollectsType<D>>,
+    collects: Mutex<CollectsType<D, QuorumJoinCert<RP>>>,
     // Used to store the finalize state when we are forced to run the CST protocol
     finalize_state: RefCell<Option<FinalizeState<D::Request>>>,
     accessory: SynchronizerAccessory<D>,
@@ -387,9 +388,11 @@ pub struct Synchronizer<D: ApplicationData> {
 /// So we protect collects, watching and tbo as those are the fields that are going to be
 /// accessed by both those threads.
 /// Since the other fields are going to be accessed by just 1 thread, we just need them to be Send, which they are
-unsafe impl<D: ApplicationData> Sync for Synchronizer<D> {}
+unsafe impl<D: ApplicationData, RP: ReconfigurationProtocolMessage> Sync for Synchronizer<D, RP> {}
 
-impl<D: ApplicationData + 'static> AbstractSynchronizer<D> for Synchronizer<D> {
+impl<D, RP> AbstractSynchronizer<D, RP> for Synchronizer<D, RP>
+    where D: ApplicationData + 'static,
+          RP: ReconfigurationProtocolMessage + 'static {
     /// Returns some information regarding the current view, such as
     /// the number of faulty replicas the system can tolerate.
     fn view(&self) -> ViewInfo {
@@ -408,14 +411,15 @@ impl<D: ApplicationData + 'static> AbstractSynchronizer<D> for Synchronizer<D> {
         }
     }
 
-    fn queue(&self, header: Header, message: ViewChangeMessage<D::Request>) {
+    fn queue(&self, header: Header, message: ViewChangeMessage<D::Request, QuorumJoinCert<RP>>) {
         self.tbo.lock().unwrap().queue(header, message)
     }
 }
 
-impl<D> Synchronizer<D>
+impl<D, RP> Synchronizer<D, RP>
     where
-        D: ApplicationData + 'static
+        D: ApplicationData + 'static,
+        RP: ReconfigurationProtocolMessage + 'static
 {
     pub fn new_follower(view: ViewInfo) -> Arc<Self> {
         Arc::new(Self {
@@ -441,12 +445,11 @@ impl<D> Synchronizer<D>
 
     /// Initialize a new `Synchronizer` with the given quorum members.
     pub fn initialize_with_quorum(seq_no: SeqNo, quorum_members: Vec<NodeId>, timeout_dur: Duration) -> Result<Arc<Self>> {
-
         let n = quorum_members.len();
 
         let f = (n - 1) / 3;
 
-        let view_info = ViewInfo::new(seq_no, n ,f)?;
+        let view_info = ViewInfo::new(seq_no, n, f)?;
 
         Ok(Arc::new(Self {
             phase: Cell::new(ProtoPhase::Init),
@@ -475,32 +478,32 @@ impl<D> Synchronizer<D>
     /// Check if we can process new view change messages.
     /// If there are pending messages that are now processable (but weren't when we received them)
     /// We return them. If there are no pending messages then we will wait for new messages from other replicas
-    pub fn poll(&self) -> SynchronizerPollStatus<D::Request> {
+    pub fn poll(&self) -> SynchronizerPollStatus<D::Request, QuorumJoinCert<RP>> {
         let mut tbo_guard = self.tbo.lock().unwrap();
         match self.phase.get() {
             _ if !tbo_guard.get_queue => SynchronizerPollStatus::Recv,
             ProtoPhase::Init => {
                 //If we are in the init phase and there is a pending request, move to the stopping phase
-                extract_msg!(D::Request =>
+                extract_msg!(D::Request, QuorumJoinCert<RP>  =>
                     { self.phase.replace(ProtoPhase::Stopping(0)); },
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop
                 )
             }
             ProtoPhase::Stopping(_) | ProtoPhase::Stopping2(_) => {
-                extract_msg!(D::Request =>
+                extract_msg!(D::Request, QuorumJoinCert<RP> =>
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop
                 )
             }
             ProtoPhase::StoppingData(_) => {
-                extract_msg!(D::Request  =>
+                extract_msg!(D::Request, QuorumJoinCert<RP>  =>
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop_data
                 )
             }
             ProtoPhase::Syncing => {
-                extract_msg!(D::Request  =>
+                extract_msg!(D::Request, QuorumJoinCert<RP>  =>
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.sync
                 )
@@ -515,17 +518,17 @@ impl<D> Synchronizer<D>
     pub fn process_message<ST, LP, NT, PL>(
         &self,
         header: Header,
-        message: ViewChangeMessage<D::Request>,
+        message: ViewChangeMessage<D::Request, QuorumJoinCert<RP>>,
         timeouts: &Timeouts,
         log: &mut Log<D, PL>,
         rq_pre_processor: &RequestPreProcessor<D::Request>,
-        consensus: &mut Consensus<D, ST, LP, PL>,
+        consensus: &mut Consensus<D, ST, LP, PL, RP>,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D>>
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
         debug!("{:?} // Processing view change message {:?} in phase {:?} from {:?}",
                node.id(),
@@ -976,9 +979,9 @@ impl<D> Synchronizer<D>
 
                 // leader has already performed this computation in the
                 // STOP-DATA phase of Mod-SMaRt
-                let signed: Vec<_> = signed_collects::<D, _, _, _>(&**node, collects);
+                let signed: Vec<_> = signed_collects::<D, _, _, _, _>(&**node, collects);
 
-                let proof = highest_proof::<D, _, _, _, _>(&current_view, &**node, signed.iter());
+                let proof = highest_proof::<D, _, _, _, _, _>(&current_view, &**node, signed.iter());
 
                 let curr_cid = proof
                     .map(|p| p.sequence_number())
@@ -1030,13 +1033,13 @@ impl<D> Synchronizer<D>
         &self,
         log: &mut Log<D, PL>,
         timeouts: &Timeouts,
-        consensus: &mut Consensus<D, ST, LP, PL>,
+        consensus: &mut Consensus<D, ST, LP, PL, RP>,
         node: &Arc<NT>,
     ) -> Option<()>
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D>>
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
         let state = self.finalize_state.borrow_mut().take()?;
 
@@ -1072,8 +1075,8 @@ impl<D> Synchronizer<D>
     )
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP>>,
-              PL: OrderingProtocolLog<PBFTConsensus<D>>
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>,
+              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
         match (self.phase.get(), &timed_out) {
             // we have received STOP messages from peer nodes,
@@ -1125,7 +1128,7 @@ impl<D> Synchronizer<D>
         _normalized_collects: Vec<Option<&CollectData<D::Request>>>,
         log: &Log<D, PL>,
     ) -> FinalizeStatus<D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>>
+        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
         let last_executed_cid = proof.as_ref().map(|p| p.sequence_number()).unwrap_or(SeqNo::ZERO);
 
@@ -1156,13 +1159,13 @@ impl<D> Synchronizer<D>
         state: FinalizeState<D::Request>,
         log: &mut Log<D, PL>,
         timeouts: &Timeouts,
-        consensus: &mut Consensus<D, ST, LP, PL>,
+        consensus: &mut Consensus<D, ST, LP, PL, RP>,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D>>
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
         let FinalizeState {
             curr_cid,
@@ -1251,7 +1254,7 @@ impl<D> Synchronizer<D>
                                         node: &NT)
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         match &self.accessory {
             SynchronizerAccessory::Follower(_) => {}
             SynchronizerAccessory::Replica(rep) => {
@@ -1264,7 +1267,7 @@ impl<D> Synchronizer<D>
     /// Requests that have timed out
     pub fn client_requests_timed_out(
         &self,
-        base_sync: &Synchronizer<D>,
+        base_sync: &Synchronizer<D, RP>,
         my_id: NodeId,
         seq: &Vec<RqTimeout>,
     ) -> SynchronizerStatus<D::Request> {
@@ -1284,7 +1287,7 @@ impl<D> Synchronizer<D>
     // may be executing the same CID when there is a leader change
     #[inline]
     fn normalized_collects<'a>(
-        collects: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request>>>,
+        collects: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
         in_exec: SeqNo,
     ) -> impl Iterator<Item=Option<&'a CollectData<D::Request>>> {
         let values = collects.values();
@@ -1297,15 +1300,15 @@ impl<D> Synchronizer<D>
     // TODO: quorum sizes may differ when we implement reconfiguration
     #[inline]
     fn highest_proof<'a, ST, LP, NT>(
-        guard: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request>>>,
+        guard: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
         view: &ViewInfo,
         node: &NT,
     ) -> Option<&'a Proof<D::Request>>
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP>>
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>
     {
-        highest_proof::<D, _, _, _, _>(&view, node, guard.values())
+        highest_proof::<D, _, _, _, _, _>(&view, node, guard.values())
     }
 }
 
@@ -1508,8 +1511,8 @@ fn certified_value<O>(
     count > curr_view.params().f()
 }
 
-fn collect_data<'a, O: 'a>(
-    collects: impl Iterator<Item=&'a StoredMessage<ViewChangeMessage<O>>>,
+fn collect_data<'a, O: 'a, JC: 'a>(
+    collects: impl Iterator<Item=&'a StoredMessage<ViewChangeMessage<O, JC>>>,
 ) -> impl Iterator<Item=&'a CollectData<O>> {
     collects.filter_map(|stored| match stored.message().kind() {
         ViewChangeMessageKind::StopData(collects) => Some(collects),
@@ -1530,28 +1533,30 @@ fn normalized_collects<'a, O: 'a>(
     })
 }
 
-fn signed_collects<D, ST, LP, NT>(
+fn signed_collects<D, ST, LP, NT, RP>(
     node: &NT,
-    collects: Vec<StoredMessage<ViewChangeMessage<D::Request>>>,
-) -> Vec<StoredMessage<ViewChangeMessage<D::Request>>>
+    collects: Vec<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
+) -> Vec<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>
     where
         D: ApplicationData + 'static,
         ST: StateTransferMessage + 'static,
         LP: LogTransferMessage + 'static,
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP>>
+        RP: ReconfigurationProtocolMessage + 'static,
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>
 {
     collects
         .into_iter()
-        .filter(|stored| validate_signature::<D, _, _, _, _>(node, stored))
+        .filter(|stored| validate_signature::<D, _, _, _, _, _>(node, stored))
         .collect()
 }
 
-fn validate_signature<'a, D, M, ST, LP, NT>(node: &'a NT, stored: &'a StoredMessage<M>) -> bool
+fn validate_signature<'a, D, M, ST, LP, NT, RP>(node: &'a NT, stored: &'a StoredMessage<M>) -> bool
     where
         D: ApplicationData + 'static,
         ST: StateTransferMessage + 'static,
         LP: LogTransferMessage + 'static,
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP>>
+        RP: ReconfigurationProtocolMessage + 'static,
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>
 {
     //TODO: Fix this as I believe it will always be false
     let wm = match WireMessage::from_header(*stored.header()) {
@@ -1577,17 +1582,18 @@ fn validate_signature<'a, D, M, ST, LP, NT>(node: &'a NT, stored: &'a StoredMess
     wm.is_valid(Some(&key), false)
 }
 
-fn highest_proof<'a, D, I, ST, LP, NT>(
+fn highest_proof<'a, D, I, ST, LP, NT, RP>(
     view: &ViewInfo,
     node: &NT,
     collects: I,
 ) -> Option<&'a Proof<D::Request>>
     where
         D: ApplicationData + 'static,
-        I: Iterator<Item=&'a StoredMessage<ViewChangeMessage<D::Request>>>,
+        I: Iterator<Item=&'a StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
+        RP: ReconfigurationProtocolMessage + 'static,
         ST: StateTransferMessage + 'static,
         LP: LogTransferMessage + 'static,
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP>>
+        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>
 {
     collect_data(collects)
         // fetch proofs
@@ -1602,13 +1608,12 @@ fn highest_proof<'a, D, I, ST, LP, NT>(
                 .iter()
                 .filter(|stored| {
                     stored.message()
-                        .consensus()
                         .has_proposed_digest(&digest)
                         //If he does not have the digest, then it is not valid
                         .unwrap_or(false)
                 })
                 .filter(move |&stored|
-                    { validate_signature::<D, _, _, _, _>(node, stored) })
+                    { validate_signature::<D, _, _, _, _, _>(node, stored) })
                 .count() >= view.params().quorum();
 
             let prepares_valid = proof
@@ -1617,13 +1622,12 @@ fn highest_proof<'a, D, I, ST, LP, NT>(
                 .filter(|stored| {
                     stored
                         .message()
-                        .consensus()
                         .has_proposed_digest(&digest)
                         //If he does not have the digest, then it is not valid
                         .unwrap_or(false)
                 })
                 .filter(move |&stored|
-                    { validate_signature::<D, _, _, _, _>(node, stored) })
+                    { validate_signature::<D, _, _, _, _, _>(node, stored) })
                 .count() >= view.params().quorum();
 
             debug!("{:?} // Proof {:?} is valid? commits valid: {:?} &&  prepares_valid: {:?}",
@@ -1635,7 +1639,7 @@ fn highest_proof<'a, D, I, ST, LP, NT>(
 }
 
 
-impl<O> Debug for SynchronizerPollStatus<O> {
+impl<O, JC> Debug for SynchronizerPollStatus<O, JC> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
             SynchronizerPollStatus::Recv => {
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 2d6bd9bc..82b791da 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -18,7 +18,7 @@ use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, RequestMessage, StoredRequestMessage, SystemMessage};
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
-use atlas_core::serialize::{LogTransferMessage, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, TimeoutPhase, Timeouts};
 use atlas_metrics::metrics::{metric_duration, metric_increment};
 use crate::bft::consensus::Consensus;
@@ -58,11 +58,11 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     ///
     /// Therefore, we start by clearing our stopped requests and treating them as
     /// newly proposed requests (by resetting their timer)
-    pub(super) fn handle_stopping_quorum<ST, LP, NT, PL>(
+    pub(super) fn handle_stopping_quorum<ST, LP, NT, PL, RP>(
         &self,
-        base_sync: &Synchronizer<D>,
+        base_sync: &Synchronizer<D, RP>,
         previous_view: ViewInfo,
-        consensus: &Consensus<D, ST, LP, PL>,
+        consensus: &Consensus<D, ST, LP, PL, RP>,
         log: &Log<D, PL>,
         pre_processor: &RequestPreProcessor<D::Request>,
         timeouts: &Timeouts,
@@ -70,8 +70,9 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     )
         where ST: StateTransferMessage + 'static,
               LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP>>,
-              PL: OrderingProtocolLog<PBFTConsensus<D>> {
+              RP: ReconfigurationProtocolMessage + 'static,
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>,
+              PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         // NOTE:
         // - install new view (i.e. update view seq no) (Done in the synchronizer)
         // - add requests from STOP into client requests
@@ -110,15 +111,16 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     /// Start a new view change
     /// Receives the requests that it should send to the other
     /// nodes in its STOP message
-    pub(super) fn handle_begin_view_change<ST, LP, NT>(
+    pub(super) fn handle_begin_view_change<ST, LP, NT, RP>(
         &self,
-        base_sync: &Synchronizer<D>,
+        base_sync: &Synchronizer<D, RP>,
         timeouts: &Timeouts,
         node: &NT,
         timed_out: Option<Vec<StoredRequestMessage<D::Request>>>,
     ) where ST: StateTransferMessage + 'static,
             LP: LogTransferMessage + 'static,
-            NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+            RP: ReconfigurationProtocolMessage + 'static,
+            NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         // stop all timers
         self.unwatch_all_requests(timeouts);
 
@@ -170,7 +172,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     ) -> Vec<ClientRqInfo> {
         let start_time = Instant::now();
 
-        let requests = match pre_prepare.message().consensus().kind() {
+        let requests = match pre_prepare.message().kind() {
             ConsensusMessageKind::PrePrepare(req) => { req }
             _ => {
                 error!("Cannot receive a request that is not a PrePrepare");
@@ -208,9 +210,10 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     }
 
     /// Register all of the requests that are missing from the view change
-    fn take_stopped_requests_and_register_them(&self, base_sync: &Synchronizer<D>,
-                                               pre_processor: &RequestPreProcessor<D::Request>,
-                                               timeouts: &Timeouts) {
+    fn take_stopped_requests_and_register_them<RP>(&self, base_sync: &Synchronizer<D, RP>,
+                                                   pre_processor: &RequestPreProcessor<D::Request>,
+                                                   timeouts: &Timeouts)
+        where RP: ReconfigurationProtocolMessage + 'static {
         // TODO: maybe optimize this `stopped_requests` call, to avoid
         // a heap allocation of a `Vec`?
 
@@ -256,12 +259,13 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     // have surpassed their defined timeout period, after the timeout event
     // is fired on the master channel of the core server task
     //
-    pub fn client_requests_timed_out(
+    pub fn client_requests_timed_out<RP>(
         &self,
-        base_sync: &Synchronizer<D>,
+        base_sync: &Synchronizer<D, RP>,
         my_id: NodeId,
         timed_out_rqs: &Vec<RqTimeout>,
-    ) -> SynchronizerStatus<D::Request> {
+    ) -> SynchronizerStatus<D::Request>
+        where RP: ReconfigurationProtocolMessage + 'static {
 
         //// iterate over list of watched pending requests,
         //// and select the ones to be stopped or forwarded
@@ -331,14 +335,15 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
 
     /// Forward the requests that timed out, `timed_out`, to all the nodes in the
     /// current view.
-    pub fn forward_requests<ST, LP, NT>(
+    pub fn forward_requests<ST, LP, NT, RP>(
         &self,
-        base_sync: &Synchronizer<D>,
+        base_sync: &Synchronizer<D, RP>,
         timed_out: Vec<StoredRequestMessage<D::Request>>,
         node: &NT,
     ) where ST: StateTransferMessage + 'static,
             LP: LogTransferMessage + 'static,
-            NT: ProtocolNetworkNode<PBFT<D, ST, LP>> {
+            RP: ReconfigurationProtocolMessage + 'static,
+            NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
         let message = SystemMessage::ForwardedRequestMessage(ForwardedRequestsMessage::new(timed_out));
         let view = base_sync.view();
 
@@ -351,11 +356,12 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     /// to other nodes
     ///
     /// Clones all the nodes in the `stopped` list
-    fn stopped_requests(
+    fn stopped_requests<RP>(
         &self,
-        base_sync: &Synchronizer<D>,
+        base_sync: &Synchronizer<D, RP>,
         requests: Option<Vec<StoredRequestMessage<D::Request>>>,
-    ) -> Vec<StoredRequestMessage<D::Request>> {
+    ) -> Vec<StoredRequestMessage<D::Request>>
+        where RP: ReconfigurationProtocolMessage + 'static {
         // Use a hashmap so we are sure we don't send any repeat requests in our stop messages
         let mut all_reqs = collections::hash_map();
 
@@ -380,11 +386,12 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         all_reqs.drain().map(|(_, stop)| stop).collect()
     }
 
-    fn stopped_request_digests(
+    fn stopped_request_digests<RP>(
         &self,
-        base_sync: &Synchronizer<D>,
+        base_sync: &Synchronizer<D, RP>,
         requests: Option<Vec<StoredRequestMessage<D::Request>>>,
-    ) -> Vec<ClientRqInfo> {
+    ) -> Vec<ClientRqInfo>
+        where RP: ReconfigurationProtocolMessage + 'static {
 
         // Use a hashmap so we are sure we don't send any repeat requests in our stop messages
         let mut all_reqs = collections::hash_set();
@@ -409,8 +416,9 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     }
 
     /// Drain our current received stopped messages
-    fn drain_stopped_request(&self, base_sync: &Synchronizer<D>) ->
-    Vec<StoredRequestMessage<D::Request>> {
+    fn drain_stopped_request<RP>(&self, base_sync: &Synchronizer<D, RP>) ->
+    Vec<StoredRequestMessage<D::Request>>
+        where RP: ReconfigurationProtocolMessage + 'static {
 
         // Use a hashmap so we are sure we don't send any repeat requests in our stop messages
         let mut all_reqs = collections::hash_map();

From 93f6bba0787b55019e06422285bf6f1d911d4a7f Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 20 Jul 2023 17:29:00 +0100
Subject: [PATCH 20/80] Added one more phase (Which is an alternative to the
 normal stop and not another stage).

---
 febft-pbft-consensus/src/bft/message/mod.rs   |  24 ++
 febft-pbft-consensus/src/bft/mod.rs           |  11 +
 febft-pbft-consensus/src/bft/sync/mod.rs      | 318 ++++++++++++++++--
 febft-pbft-consensus/src/bft/sync/view/mod.rs |   8 +
 4 files changed, 339 insertions(+), 22 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/message/mod.rs b/febft-pbft-consensus/src/bft/message/mod.rs
index 64a0d704..4258edb1 100644
--- a/febft-pbft-consensus/src/bft/message/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/mod.rs
@@ -185,8 +185,13 @@ impl<O, JC> ViewChangeMessage<O, JC> {
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
 pub enum ViewChangeMessageKind<O, JC> {
+    /// A message, broadcast by the node attempting to join the quorum
     NodeQuorumJoin(NodeId, JC),
+    /// A STOP message, broadcast when we want to call a view change due to requests getting timed out
     Stop(Vec<StoredRequestMessage<O>>),
+    /// A STOP message, broadcast when we want to call a view change due to us having received a Node Quorum Join message
+    StopQuorumJoin(NodeId, JC),
+    // Each of the latest decisions from the sender, so the new leader can sync
     StopData(CollectData<O>),
     Sync(LeaderCollects<O, JC>),
 }
@@ -441,3 +446,22 @@ impl Debug for ObserveEventKind {
         }
     }
 }
+
+impl<O, JC> Debug for ViewChangeMessageKind<O, JC> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            ViewChangeMessageKind::NodeQuorumJoin(node, _) => {
+                write!(f, "Node quorum join {:?}", node)
+            }
+            ViewChangeMessageKind::Stop(_) => {
+                write!(f, "Stop message")
+            }
+            ViewChangeMessageKind::StopData(_) => {
+                write!(f, "Stop data message")
+            }
+            ViewChangeMessageKind::Sync(_) => {
+                write!(f, "Sync message")
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index aa0912c2..4741d97d 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -27,6 +27,7 @@ use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::messages::Protocol;
 use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, ProtocolConsensusDecision, ProtocolMessage, SerProof, SerProofMetadata, View};
+use atlas_core::ordering_protocol::reconfigurable_order_protocol::{ReconfigurableOrderProtocol, ReconfigurationAttemptResult};
 use atlas_core::ordering_protocol::stateful_order_protocol::{DecLog, StatefulOrderProtocol};
 use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol, StatefulOrderingProtocolLog};
 use atlas_core::reconfiguration_protocol::{QuorumJoinCert, ReconfigurationProtocol};
@@ -813,3 +814,13 @@ impl<D, ST, LP, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTC
         proofs.proofs().iter().collect()
     }
 }
+
+impl<D, ST, LP, NT, PL, RP> ReconfigurableOrderProtocol<RP, NT> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
+    where D: ApplicationData + 'static,
+          ST: StateTransferMessage + 'static,
+          LP: LogTransferMessage + 'static, {
+    fn attempt_network_view_change(&mut self, join_certificate: QuorumJoinCert<RP>) -> Result<ReconfigurationAttemptResult> {
+
+        Ok(self.synchronizer.attempt_join_quorum(join_certificate, &self.node, &self.timeouts))
+    }
+}
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index f7ac36d2..ee7420ab 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -6,6 +6,7 @@ use std::{
     time::Duration,
 };
 use std::cell::Cell;
+use std::collections::{BTreeMap, BTreeSet};
 use std::fmt::{Debug, Formatter};
 
 use either::Either;
@@ -24,6 +25,7 @@ use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
+use atlas_core::ordering_protocol::reconfigurable_order_protocol::ReconfigurationAttemptResult;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::reconfiguration_protocol::QuorumJoinCert;
 use atlas_core::request_pre_processing::RequestPreProcessor;
@@ -177,6 +179,8 @@ pub struct TboQueue<O, JC> {
     // probe messages from this queue instead of
     // fetching them from the network
     get_queue: bool,
+    // stores all Node Quorum Update messages for the next view
+    quorum_join: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O, JC>>>>,
     // stores all STOP messages for the next view
     stop: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O, JC>>>>,
     // stores all STOP-DATA messages for the next view
@@ -191,6 +195,7 @@ impl<O, JC> TboQueue<O, JC> {
             view,
             previous_view: None,
             get_queue: false,
+            quorum_join: VecDeque::new(),
             stop: VecDeque::new(),
             stop_data: VecDeque::new(),
             sync: VecDeque::new(),
@@ -231,6 +236,7 @@ impl<O, JC> TboQueue<O, JC> {
     }
 
     fn next_instance_queue(&mut self) {
+        tbo_advance_message_queue(&mut self.quorum_join);
         tbo_advance_message_queue(&mut self.stop);
         tbo_advance_message_queue(&mut self.stop_data);
         tbo_advance_message_queue(&mut self.sync);
@@ -240,9 +246,12 @@ impl<O, JC> TboQueue<O, JC> {
     /// immediately if it pertains to an older view change instance.
     pub fn queue(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
         match m.kind() {
-            ViewChangeMessageKind::Stop(_) => self.queue_stop(h, m),
+            ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) => self.queue_stop(h, m),
             ViewChangeMessageKind::StopData(_) => self.queue_stop_data(h, m),
             ViewChangeMessageKind::Sync(_) => self.queue_sync(h, m),
+            ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
+                self.queue_node_quorum_join(h, m)
+            }
         }
     }
 
@@ -255,6 +264,12 @@ impl<O, JC> TboQueue<O, JC> {
             .unwrap_or(false)
     }
 
+    fn queue_node_quorum_join(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
+        let seq = self.view.sequence_number().next();
+
+        tbo_queue_message(seq, &mut self.quorum_join, StoredMessage::new(h, m))
+    }
+
     /// Queues a `STOP` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
     fn queue_stop(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
@@ -303,6 +318,12 @@ pub(super) enum ProtoPhase {
     // this is effectively an implementation detail,
     // and not a real phase of Mod-SMaRt!
     Stopping2(usize),
+    // we are running the STOP-QUORUM-JOIN phase
+    ViewStopping(usize),
+    // we are running the STOP-QUORUM-JOIN phase
+    // but we have already locally triggered the view change
+    // Or we have received at least f+1 STOP-QUORUM-JOIN messages.
+    ViewStopping2(usize),
     // we are running the STOP-DATA phase of Mod-SMaRt
     StoppingData(usize),
     // we are running the SYNC phase of Mod-SMaRt
@@ -374,6 +395,8 @@ pub struct Synchronizer<D: ApplicationData, RP: ReconfigurationProtocolMessage>
     tbo: Mutex<TboQueue<D::Request, QuorumJoinCert<RP>>>,
     //Stores currently received requests from other nodes
     stopped: RefCell<IntMap<Vec<StoredRequestMessage<D::Request>>>>,
+    //Stores currently received requests from other nodes
+    currently_adding_node: Cell<Option<NodeId>>,
     //TODO: This does not require a Mutex I believe since it's only accessed when
     // Processing messages (which is always done in the replica thread)
     collects: Mutex<CollectsType<D, QuorumJoinCert<RP>>>,
@@ -425,6 +448,7 @@ impl<D, RP> Synchronizer<D, RP>
         Arc::new(Self {
             phase: Cell::new(ProtoPhase::Init),
             stopped: RefCell::new(Default::default()),
+            currently_adding_node: Cell::new(None),
             collects: Mutex::new(Default::default()),
             tbo: Mutex::new(TboQueue::new(view)),
             finalize_state: RefCell::new(None),
@@ -436,6 +460,7 @@ impl<D, RP> Synchronizer<D, RP>
         Arc::new(Self {
             phase: Cell::new(ProtoPhase::Init),
             stopped: RefCell::new(Default::default()),
+            currently_adding_node: Cell::new(None),
             collects: Mutex::new(Default::default()),
             tbo: Mutex::new(TboQueue::new(view)),
             finalize_state: RefCell::new(None),
@@ -455,6 +480,7 @@ impl<D, RP> Synchronizer<D, RP>
             phase: Cell::new(ProtoPhase::Init),
             tbo: Mutex::new(TboQueue::new(view_info)),
             stopped: RefCell::new(Default::default()),
+            currently_adding_node: Cell::new(None),
             collects: Mutex::new(Default::default()),
             finalize_state: RefCell::new(None),
             accessory: SynchronizerAccessory::Replica(ReplicaSynchronizer::new(timeout_dur)),
@@ -484,17 +510,28 @@ impl<D, RP> Synchronizer<D, RP>
             _ if !tbo_guard.get_queue => SynchronizerPollStatus::Recv,
             ProtoPhase::Init => {
                 //If we are in the init phase and there is a pending request, move to the stopping phase
-                extract_msg!(D::Request, QuorumJoinCert<RP>  =>
+                let result = extract_msg!(D::Request, QuorumJoinCert<RP>  =>
                     { self.phase.replace(ProtoPhase::Stopping(0)); },
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop
-                )
+                );
+
+                match result {
+                    SynchronizerPollStatus::Recv => {
+                        // if we don't have any pending stop messages, check the join quorum
+                        extract_msg!(D::Request, QuorumJoinCert<RP> => {self.phase.replace(ProtoPhase::ViewStopping(0));},
+                            &mut tbo_guard.get_queue, &mut tbo_guard.quorum_join)
+                    }
+                    _ => {
+                        result
+                    }
+                }
             }
-            ProtoPhase::Stopping(_) | ProtoPhase::Stopping2(_) => {
-                extract_msg!(D::Request, QuorumJoinCert<RP> =>
+            ProtoPhase::Stopping(_) | ProtoPhase::Stopping2(_) | ProtoPhase::ViewStopping(_) | ProtoPhase::ViewStopping2(_) => {
+                let result = extract_msg!(D::Request, QuorumJoinCert<RP> =>
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop
-                )
+                );
             }
             ProtoPhase::StoppingData(_) => {
                 extract_msg!(D::Request, QuorumJoinCert<RP>  =>
@@ -539,12 +576,21 @@ impl<D, RP> Synchronizer<D, RP>
         match self.phase.get() {
             ProtoPhase::Init => {
                 return match message.kind() {
-                    ViewChangeMessageKind::Stop(_) => {
+                    ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
+                        let mut guard = self.tbo.lock().unwrap();
+
+                        guard.queue_node_join(header, message);
+
+                        debug!("{:?} // Received {:?} message while in init state. Queueing", message.kind(), node.id());
+
+                        SynchronizerStatus::Nil
+                    }
+                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) => {
                         let mut guard = self.tbo.lock().unwrap();
 
                         guard.queue_stop(header, message);
 
-                        debug!("{:?} // Received stop message while in init state. Queueing", node.id());
+                        debug!("{:?} // Received {:?} message while in init state. Queueing", message.kind(), node.id());
 
                         SynchronizerStatus::Nil
                     }
@@ -582,7 +628,16 @@ impl<D, RP> Synchronizer<D, RP>
                 let next_seq = current_view.sequence_number().next();
 
                 let i = match message.kind() {
-                    ViewChangeMessageKind::Stop(_) if msg_seq != next_seq => {
+                    ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
+                        let mut guard = self.tbo.lock().unwrap();
+
+                        guard.queue_node_join(header, message);
+
+                        debug!("{:?} // Received node join message while in stopping state. Queueing", node.id());
+
+                        return stop_status!(i, &current_view);
+                    }
+                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) if msg_seq != next_seq => {
                         debug!("{:?} // Received stop message {:?} that does not match up to our local view {:?}", node.id(), message, current_view);
 
                         let mut guard = self.tbo.lock().unwrap();
@@ -591,14 +646,17 @@ impl<D, RP> Synchronizer<D, RP>
 
                         return stop_status!(i, &current_view);
                     }
-                    ViewChangeMessageKind::Stop(_)
-                    if self.stopped.borrow().contains_key(header.from().into()) =>
-                        {
-                            warn!("{:?} // Received double stop message from node {:?}", node.id(), header.from());
+                    ViewChangeMessageKind::Stop(_) if self.stopped.borrow().contains_key(header.from().into()) => {
+                        warn!("{:?} // Received double stop message from node {:?}", node.id(), header.from());
 
-                            // drop attempts to vote twice
-                            return stop_status!(i, &current_view);
-                        }
+                        // drop attempts to vote twice
+                        return stop_status!(i, &current_view);
+                    }
+                    ViewChangeMessageKind::StopQuorumJoin(_, _) => {
+                        warn!("{:?} // Received stop quorum join message while in stopping state. Ignoring", node.id());
+
+                        return stop_status!(i, &current_view);
+                    }
                     ViewChangeMessageKind::Stop(_) => i + 1,
                     ViewChangeMessageKind::StopData(_) => {
                         match &self.accessory {
@@ -693,6 +751,130 @@ impl<D, RP> Synchronizer<D, RP>
 
                 SynchronizerStatus::Running
             }
+            ProtoPhase::ViewStopping(received) | ProtoPhase::ViewStopping2(received) => {
+                let msg_seq = message.sequence_number();
+                let current_view = self.view();
+                let next_seq = current_view.sequence_number().next();
+
+                let (received, node, jc) = match message.kind() {
+                    ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
+                        let mut guard = self.tbo.lock().unwrap();
+
+                        guard.queue_node_join(header, message);
+
+                        debug!("{:?} // Received node join message while in stopping state. Queueing", node.id());
+
+                        return stop_status!(received, &current_view);
+                    }
+                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) if msg_seq != next_seq => {
+                        debug!("{:?} // Received stop message {:?} that does not match up to our local view {:?}", node.id(), message, current_view);
+
+                        let mut guard = self.tbo.lock().unwrap();
+
+                        guard.queue_stop(header, message);
+
+                        return stop_status!(received, &current_view);
+                    }
+                    ViewChangeMessageKind::Stop(requests) => {
+                        let mut guard = self.tbo.lock().unwrap();
+
+                        warn!("{:?} // Received stop message while in View Stopping state. Since STOP takes precendence over the quorum updating, we will now change to stopping phase ", node.id());
+
+                        guard.queue_stop(header, message);
+
+                        self.phase.replace(ProtoPhase::Stopping(0));
+
+                        return SynchronizerStatus::Running;
+                    }
+                    ViewChangeMessageKind::StopQuorumJoin(node, join_cert)
+                    if self.currently_adding_node.get().is_some() && self.currently_adding_node.get().unwrap() != *node => {
+                        error!("{:?} // Received stop quorum join message with node that does not match the current received ", node.id());
+
+                        return SynchronizerStatus::Running;
+                    }
+                    ViewChangeMessageKind::StopQuorumJoin(node, join_cert) => (received + 1, *node, join_cert),
+                    ViewChangeMessageKind::StopData(_) => {
+                        match &self.accessory {
+                            SynchronizerAccessory::Follower(_) => {
+                                //Ignore stop data messages as followers can never reach this state
+                                return stop_status!(received, &current_view);
+                            }
+                            SynchronizerAccessory::Replica(_) => {
+                                {
+                                    let mut guard = self.tbo.lock().unwrap();
+
+                                    guard.queue_stop_data(header, message);
+
+                                    debug!("{:?} // Received stop data message while in stopping state. Queueing", node.id());
+                                }
+
+                                return stop_status!(received, &current_view);
+                            }
+                        }
+                    }
+                    ViewChangeMessageKind::Sync(_) => {
+                        {
+                            let mut guard = self.tbo.lock().unwrap();
+
+                            guard.queue_sync(header, message);
+
+                            debug!("{:?} // Received sync message while in init state. Queueing", node.id());
+                        }
+
+                        return stop_status!(received, &current_view);
+                    }
+                };
+
+                //TODO: Verify that the node join certificate is valid
+
+                if let ProtoPhase::ViewStopping(_received) = self.phase.get() {
+                    return if received > current_view.params().f() {
+                        self.begin_quorum_view_change(None, &**node, timeouts, log);
+
+                        SynchronizerStatus::Running
+                    } else {
+                        self.phase.replace(ProtoPhase::ViewStopping(received));
+
+                        SynchronizerStatus::Nil
+                    };
+                }
+
+                if received >= current_view.params().quorum() {
+                    let next_view = current_view.next_view_with_new_node(self.currently_adding_node.get().unwrap());
+
+                    let previous_view = current_view.clone();
+
+                    //We have received the necessary amount of stopping requests
+                    //To now that we should move to the next view
+
+                    let next_leader = next_view.leader();
+
+                    warn!("{:?} // Stopping quorum reached, moving to next view {:?}. ", node.id(), next_view);
+
+                    self.install_view(next_view);
+
+                    match &self.accessory {
+                        SynchronizerAccessory::Replica(rep) => {
+                            rep.handle_stopping_quorum(self, previous_view, consensus,
+                                                       log, rq_pre_processor, timeouts, &**node)
+                        }
+                        SynchronizerAccessory::Follower(_) => {}
+                    }
+
+                    if next_leader == node.id() {
+                        warn!("{:?} // I am the new leader, moving to the stopping data phase.", node.id());
+
+                        //Move to the stopping data phase as we are the new leader
+                        self.phase.replace(ProtoPhase::StoppingData(0));
+                    } else {
+                        self.phase.replace(ProtoPhase::Syncing);
+                    }
+                } else {
+                    self.phase.replace(ProtoPhase::ViewStopping2(received));
+                }
+
+                SynchronizerStatus::Running
+            }
             ProtoPhase::StoppingData(i) => {
                 match &self.accessory {
                     SynchronizerAccessory::Follower(_) => {
@@ -711,7 +893,18 @@ impl<D, RP> Synchronizer<D, RP>
                         let mut collects_guard = self.collects.lock().unwrap();
 
                         let i = match message.kind() {
-                            ViewChangeMessageKind::Stop(_) => {
+                            ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
+                                {
+                                    let mut guard = self.tbo.lock().unwrap();
+
+                                    guard.queue_node_quorum_join(header, message);
+
+                                    debug!("{:?} // Received stop message while in stopping data state. Queueing", node.id());
+                                }
+
+                                return SynchronizerStatus::Running;
+                            }
+                            ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) => {
                                 {
                                     let mut guard = self.tbo.lock().unwrap();
 
@@ -788,7 +981,7 @@ impl<D, RP> Synchronizer<D, RP>
                         collects_guard
                             .insert(header.from().into(), StoredMessage::new(header, message));
 
-                        if i != current_view.params().quorum() {
+                        if i < current_view.params().quorum() {
                             self.phase.replace(ProtoPhase::StoppingData(i));
 
                             return SynchronizerStatus::Running;
@@ -1060,6 +1253,74 @@ impl<D, RP> Synchronizer<D, RP>
         Some(())
     }
 
+    pub fn begin_quorum_view_change<ST, LP, NT, PL>(
+        &self,
+        join_cert: Option<QuorumJoinCert<RP>>,
+        node: &NT,
+        timeouts: &Timeouts,
+        _log: &Log<D, PL>, )
+        where ST: StateTransferMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>,
+              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
+    {
+        match (self.phase.get(), &join_cert) {
+            (ProtoPhase::ViewStopping(i), None) => {
+                self.phase.replace(ProtoPhase::ViewStopping2(i + 1));
+            }
+            (ProtoPhase::ViewStopping(i), _) => {
+                // We have ourselves received a join certificate message from the node, so we will now
+                // Have to broadcast a STOP Quorum Join message. As such, we don't want to increment
+                // The amount of messages received (since we have not actually received any messages)
+                self.phase.replace(ProtoPhase::ViewStopping2(i));
+            }
+            (ProtoPhase::StoppingData(_), _) | (ProtoPhase::Syncing, _) | (ProtoPhase::Stopping2(_), _) => {
+                // we have already started a view change protocol
+                return;
+            }
+            _ => {
+                self.node_join_certificates.borrow_mut().clear();
+            }
+        }
+    }
+
+    pub fn attempt_join_quorum<ST, LP, NT>(&self, join_certificate: QuorumJoinCert<RP>,
+                                           node: &NT,
+                                           timeouts: &Timeouts) -> ReconfigurationAttemptResult
+        where ST: StateTransferMessage + 'static,
+              LP: LogTransferMessage + 'static,
+              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>, {
+        let current_view = self.view();
+
+        if current_view.quorum_members().contains(&node.id()) {
+            //We are already a part of the quorum, so we don't need to do anything
+            info!("{:?} // Attempted to join quorum, but we are already a part of it", node.id());
+            return ReconfigurationAttemptResult::AlreadyPartOfQuorum;
+        }
+
+        let message = ViewChangeMessage::new(
+            current_view.sequence_number().next(),
+            ViewChangeMessageKind::NodeQuorumJoin(node.id(), join_certificate),
+        );
+
+        node.broadcast_signed(PBFTMessage::ViewChange(message), current_view.quorum_members().clone().into_iter());
+
+        // Simulate that we were accepted into the quorum
+        let view = current_view.next_view_with_new_node(node.id());
+
+        if view.leader() == node.id() {
+            // If we are the leader of the next view, then we should move to the stopping data phase and wait
+            // For the rest of the nodes to send us the information
+            self.phase.replace(ProtoPhase::StoppingData(0));
+        } else {
+            self.phase.replace(ProtoPhase::Syncing);
+        }
+
+        self.install_view(view);
+
+        return ReconfigurationAttemptResult::InProgress;
+    }
+
     /// Trigger a view change locally.
     ///
     /// The value `timed_out` corresponds to a list of client requests
@@ -1092,6 +1353,18 @@ impl<D, RP> Synchronizer<D, RP>
             (ProtoPhase::Stopping(i), _) => {
                 self.phase.replace(ProtoPhase::Stopping2(i));
             }
+            (ProtoPhase::ViewStopping(_), None) | (ProtoPhase::ViewStopping2(_), None) => {
+                // We are currently in the quorum alteration view change protocol. Since we have received
+                // A timeout and we know that timeouts take precedence over quorum alteration, we will
+                // Stop this current view stopping protocol and start the normal view change protocol
+                self.phase.replace(ProtoPhase::Stopping(1));
+            }
+            (ProtoPhase::ViewStopping(_), _) | (ProtoPhase::ViewStopping2(_), _) => {
+                // We are currently in the quorum alteration view change protocol. Since we have received
+                // A timeout and we know that timeouts take precedence over quorum alteration, we will
+                // Stop this current view stopping protocol and start the normal view change protocol
+                self.phase.replace(ProtoPhase::Stopping2(0));
+            }
             (ProtoPhase::StoppingData(_), _) | (ProtoPhase::Syncing, _) | (ProtoPhase::Stopping2(_), _) => {
                 // we have already started a view change protocol
                 return;
@@ -1105,6 +1378,7 @@ impl<D, RP> Synchronizer<D, RP>
                 // clear state from previous views
                 self.stopped.borrow_mut().clear();
                 self.collects.lock().unwrap().clear();
+                self.currently_adding_node.replace(None);
 
                 //Set the new state to be stopping
                 self.phase.replace(ProtoPhase::Stopping2(0));
@@ -1120,7 +1394,7 @@ impl<D, RP> Synchronizer<D, RP>
     }
 
     // this function mostly serves the purpose of consuming
-    // values with immutable references, to allow borrowing data mutably
+// values with immutable references, to allow borrowing data mutably
     fn pre_finalize<PL>(
         &self,
         state: FinalizeState<D::Request>,
@@ -1282,9 +1556,9 @@ impl<D, RP> Synchronizer<D, RP>
     }
 
     // collects whose in execution cid is different from the given `in_exec` become `None`
-    // A set of collects is considered normalized if or when
-    // all collects are related to the same CID. This is important because not all replicas
-    // may be executing the same CID when there is a leader change
+// A set of collects is considered normalized if or when
+// all collects are related to the same CID. This is important because not all replicas
+// may be executing the same CID when there is a leader change
     #[inline]
     fn normalized_collects<'a>(
         collects: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
diff --git a/febft-pbft-consensus/src/bft/sync/view/mod.rs b/febft-pbft-consensus/src/bft/sync/view/mod.rs
index f0b2736d..e47e3595 100644
--- a/febft-pbft-consensus/src/bft/sync/view/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/view/mod.rs
@@ -153,6 +153,14 @@ impl ViewInfo {
         Self::new(self.seq.next(), self.params.n(), self.params.f()).unwrap()
     }
 
+    pub fn next_view_with_new_node(&self, joined_node: NodeId) -> ViewInfo {
+        let mut quorum_members = self.quorum_members().clone();
+
+        quorum_members.push(joined_node);
+
+        Self::from_quorum(self.seq.next(), quorum_members).unwrap()
+    }
+
     /// Returns a new view with the specified sequence number.
     pub fn peek(&self, seq: SeqNo) -> ViewInfo {
         Self::new(seq, self.params.n(), self.params.f()).unwrap()

From 16ef4d85c3e518cad76f12c8c31bf67820e35868 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 20 Jul 2023 23:58:15 +0100
Subject: [PATCH 21/80] Wrote the logic for quorum entering and handout to the
 ordering protocol for actual approval by the quorum (through which ever
 method the ordering protocol finds appropriate)

---
 febft-pbft-consensus/src/bft/message/mod.rs | 20 +++--------
 febft-pbft-consensus/src/bft/mod.rs         | 13 ++++---
 febft-pbft-consensus/src/bft/sync/mod.rs    | 38 +++++++++++++--------
 3 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/message/mod.rs b/febft-pbft-consensus/src/bft/message/mod.rs
index 4258edb1..5e99114e 100644
--- a/febft-pbft-consensus/src/bft/message/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/mod.rs
@@ -135,22 +135,7 @@ impl<O, JC> Orderable for ViewChangeMessage<O, JC> {
 
 impl<O, JC> Debug for ViewChangeMessage<O, JC> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        write!(f, "View {:?}", self.view)?;
-
-        match self.kind {
-            ViewChangeMessageKind::Stop(_) => {
-                write!(f, "Stop Message")
-            }
-            ViewChangeMessageKind::StopData(_) => {
-                write!(f, "Stop Data Message")
-            }
-            ViewChangeMessageKind::Sync(_) => {
-                write!(f, "Sync Message")
-            }
-            ViewChangeMessageKind::NodeQuorumJoin(node_id, _) => {
-                write!(f, "Node quorum join {:?}", node_id)
-            }
-        }
+        write!(f, "View {:?}, {:?}", self.view, self.kind)
     }
 }
 
@@ -462,6 +447,9 @@ impl<O, JC> Debug for ViewChangeMessageKind<O, JC> {
             ViewChangeMessageKind::Sync(_) => {
                 write!(f, "Sync message")
             }
+            ViewChangeMessageKind::StopQuorumJoin(node, _) => {
+                write!(f, "Stop quorum join message {:?}", node)
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 4741d97d..798d23a5 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -283,9 +283,10 @@ impl<D, ST, LP, NT, PL, RP> PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
         } = config;
 
         let OrderingProtocolArgs(executor, timeouts,
-                                 pre_processor, batch_input, node, persistent_log) = args;
+                                 pre_processor, batch_input,
+                                 node, persistent_log, quorum) = args;
 
-        let sync = Synchronizer::new_replica(view.clone(), timeout_dur);
+        let sync = Synchronizer::initialize_with_quorum(view.sequence_number(), quorum, timeout_dur)?;
 
         let consensus_guard = ProposerConsensusGuard::new(view.clone(), watermark);
 
@@ -818,9 +819,11 @@ impl<D, ST, LP, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTC
 impl<D, ST, LP, NT, PL, RP> ReconfigurableOrderProtocol<RP, NT> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static, {
+          LP: LogTransferMessage + 'static,
+          RP: ReconfigurationProtocolMessage + 'static,
+          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+          PL: Clone {
     fn attempt_network_view_change(&mut self, join_certificate: QuorumJoinCert<RP>) -> Result<ReconfigurationAttemptResult> {
-
-        Ok(self.synchronizer.attempt_join_quorum(join_certificate, &self.node, &self.timeouts))
+        Ok(self.synchronizer.attempt_join_quorum(join_certificate, &*self.node, &self.timeouts))
     }
 }
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index ee7420ab..89046e1f 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -528,10 +528,10 @@ impl<D, RP> Synchronizer<D, RP>
                 }
             }
             ProtoPhase::Stopping(_) | ProtoPhase::Stopping2(_) | ProtoPhase::ViewStopping(_) | ProtoPhase::ViewStopping2(_) => {
-                let result = extract_msg!(D::Request, QuorumJoinCert<RP> =>
+                extract_msg!(D::Request, QuorumJoinCert<RP> =>
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop
-                );
+                )
             }
             ProtoPhase::StoppingData(_) => {
                 extract_msg!(D::Request, QuorumJoinCert<RP>  =>
@@ -579,18 +579,18 @@ impl<D, RP> Synchronizer<D, RP>
                     ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
                         let mut guard = self.tbo.lock().unwrap();
 
-                        guard.queue_node_join(header, message);
+                        debug!("{:?} // Received {:?} message while in init state. Queueing",  node.id(), message);
 
-                        debug!("{:?} // Received {:?} message while in init state. Queueing", message.kind(), node.id());
+                        guard.queue_node_quorum_join(header, message);
 
                         SynchronizerStatus::Nil
                     }
                     ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) => {
                         let mut guard = self.tbo.lock().unwrap();
 
-                        guard.queue_stop(header, message);
+                        debug!("{:?} // Received {:?} message while in init state. Queueing", node.id(), message);
 
-                        debug!("{:?} // Received {:?} message while in init state. Queueing", message.kind(), node.id());
+                        guard.queue_stop(header, message);
 
                         SynchronizerStatus::Nil
                     }
@@ -602,11 +602,10 @@ impl<D, RP> Synchronizer<D, RP>
                             }
                             SynchronizerAccessory::Replica(_) => {
                                 let mut guard = self.tbo.lock().unwrap();
+                                debug!("{:?} // Received stop data {:?} message while in init state. Queueing", node.id(), message);
 
                                 guard.queue_stop_data(header, message);
 
-                                debug!("{:?} // Received stop data message while in init state. Queueing", node.id());
-
                                 SynchronizerStatus::Nil
                             }
                         }
@@ -631,7 +630,7 @@ impl<D, RP> Synchronizer<D, RP>
                     ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
                         let mut guard = self.tbo.lock().unwrap();
 
-                        guard.queue_node_join(header, message);
+                        guard.queue_node_quorum_join(header, message);
 
                         debug!("{:?} // Received node join message while in stopping state. Queueing", node.id());
 
@@ -756,11 +755,11 @@ impl<D, RP> Synchronizer<D, RP>
                 let current_view = self.view();
                 let next_seq = current_view.sequence_number().next();
 
-                let (received, node, jc) = match message.kind() {
+                let (received, node_id, jc) = match message.kind() {
                     ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
                         let mut guard = self.tbo.lock().unwrap();
 
-                        guard.queue_node_join(header, message);
+                        guard.queue_node_quorum_join(header, message);
 
                         debug!("{:?} // Received node join message while in stopping state. Queueing", node.id());
 
@@ -1115,7 +1114,18 @@ impl<D, RP> Synchronizer<D, RP>
 
                 // reject SYNC messages if these were not sent by the leader
                 let (proposed, collects) = match message.kind() {
-                    ViewChangeMessageKind::Stop(_) => {
+                    ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
+                        {
+                            let mut guard = self.tbo.lock().unwrap();
+
+                            guard.queue_node_quorum_join(header, message);
+
+                            debug!("{:?} // Received node join message while in syncing state. Queueing", node.id());
+                        }
+
+                        return SynchronizerStatus::Running;
+                    }
+                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) => {
                         {
                             let mut guard = self.tbo.lock().unwrap();
 
@@ -1279,7 +1289,7 @@ impl<D, RP> Synchronizer<D, RP>
                 return;
             }
             _ => {
-                self.node_join_certificates.borrow_mut().clear();
+                self.currently_adding_node.replace(None);
             }
         }
     }
@@ -1303,7 +1313,7 @@ impl<D, RP> Synchronizer<D, RP>
             ViewChangeMessageKind::NodeQuorumJoin(node.id(), join_certificate),
         );
 
-        node.broadcast_signed(PBFTMessage::ViewChange(message), current_view.quorum_members().clone().into_iter());
+        node.broadcast_signed(SystemMessage::from_protocol_message(PBFTMessage::ViewChange(message)), current_view.quorum_members().clone().into_iter());
 
         // Simulate that we were accepted into the quorum
         let view = current_view.next_view_with_new_node(node.id());

From acc4e906c739924166ae1e6a208e6f801be7ec68 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Sun, 23 Jul 2023 23:10:00 +0100
Subject: [PATCH 22/80] Fixed some more bugs with reconfiguration. It's kind of
 working, but still have to figure out the bootstrap process with < 3f+1 (f=1)
 nodes as bootstrap nodes. Atm just works with at least 4 nodes as bootstrap
 nodes.

---
 febft-pbft-consensus/src/bft/mod.rs | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 798d23a5..2f8e40d6 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -26,7 +26,7 @@ use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::messages::Protocol;
-use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, ProtocolConsensusDecision, ProtocolMessage, SerProof, SerProofMetadata, View};
+use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, ProtocolConsensusDecision, ProtocolMessage, SerProof, SerProofMetadata, View};
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::{ReconfigurableOrderProtocol, ReconfigurationAttemptResult};
 use atlas_core::ordering_protocol::stateful_order_protocol::{DecLog, StatefulOrderProtocol};
 use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol, StatefulOrderingProtocolLog};
@@ -126,6 +126,12 @@ impl<D, ST, LP, NT, PL, RP> Orderable for PBFTOrderProtocol<D, ST, LP, NT, PL, R
     }
 }
 
+impl<D, ST, LP, NT, PL, RP> OrderProtocolTolerance for PBFTOrderProtocol<D, ST, LP, NT, PL, RP> where D: 'static + ApplicationData, LP: 'static + LogTransferMessage, NT: 'static + ProtocolNetworkNode<PBFT<D, ST, LP, RP>>, PL: Clone, RP: 'static + ReconfigurationProtocolMessage, ST: 'static + StateTransferMessage {
+    fn get_n_for_f(f: usize) -> usize {
+        3 * f + 1
+    }
+}
+
 impl<D, ST, LP, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
@@ -286,7 +292,7 @@ impl<D, ST, LP, NT, PL, RP> PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
                                  pre_processor, batch_input,
                                  node, persistent_log, quorum) = args;
 
-        let sync = Synchronizer::initialize_with_quorum(view.sequence_number(), quorum, timeout_dur)?;
+        let sync = Synchronizer::initialize_with_quorum(view.sequence_number(), quorum.clone(), timeout_dur)?;
 
         let consensus_guard = ProposerConsensusGuard::new(view.clone(), watermark);
 
@@ -324,8 +330,8 @@ impl<D, ST, LP, NT, PL, RP> PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
 
         info!("{:?} // Watermark: {}", replica.node.id(), watermark);
 
-        println!("{:?} // Leader count: {}, Leader set: {:?}", replica.node.id(),
-                 crr_view.leader_set().len(), crr_view.leader_set());
+        println!("{:?} // Leader count: {}, Leader set: {:?}, Quorum: {:?}", replica.node.id(),
+                 crr_view.leader_set().len(), crr_view.leader_set(), quorum);
         println!("{:?} // Watermark: {}", replica.node.id(), watermark);
 
         replica.proposer.clone().start();

From 55fab07e0d67caabfc905da8018cac47373008da Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 24 Jul 2023 15:10:27 +0100
Subject: [PATCH 23/80] Started working on partitioning the Node implementation
 in order to reduce the amount of needed generics.

---
 febft-state-transfer/src/lib.rs | 130 ++++++++++----------------------
 1 file changed, 40 insertions(+), 90 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 2aec064c..a3c493c7 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -28,6 +28,7 @@ use atlas_core::persistent_log::{MonolithicStateLog, PersistableStateTransferPro
 use atlas_core::serialize::{LogTransferMessage, NetworkView, OrderingProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage, StateTransferMessage};
 use atlas_core::state_transfer::{Checkpoint, CstM, StateTransferProtocol, STResult, STTimeoutResult};
 use atlas_core::state_transfer::monolithic_state::MonolithicStateTransfer;
+use atlas_core::state_transfer::networking::StateTransferSendNode;
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, Timeouts};
 use atlas_execution::state::monolithic_state::{InstallStateMessage, MonolithicState};
 use atlas_metrics::metrics::metric_duration;
@@ -230,28 +231,21 @@ macro_rules! getmessage {
 
 impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT, PL>
     where S: MonolithicState + 'static,
-          PL: MonolithicStateLog<S> + 'static
+          PL: MonolithicStateLog<S> + 'static,
+          NT: StateTransferSendNode<CSTMsg<S>> + 'static
 {
     type Serialization = CSTMsg<S>;
 
-    fn request_latest_state<D, OP, LP, V>(&mut self, view: V) -> Result<()>
-        where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>,
-              V: NetworkView {
-        self.request_latest_consensus_seq_no::<D, OP, LP, V>(view);
+    fn request_latest_state<V>(&mut self, view: V) -> Result<()>
+        where V: NetworkView {
+        self.request_latest_consensus_seq_no::<V>(view);
 
         Ok(())
     }
 
-    fn handle_off_ctx_message<D, OP, LP, V>(&mut self, view: V, message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
-                                            -> Result<()>
-        where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>,
-              V: NetworkView {
+    fn handle_off_ctx_message<V>(&mut self, view: V, message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
+                                 -> Result<()>
+        where V: NetworkView {
         let (header, message) = message.into_inner();
 
         let message = message.into_inner();
@@ -288,15 +282,9 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         Ok(())
     }
 
-    fn process_message<D, OP, LP, V>(&mut self,
-                                     view: V,
-                                     message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
-                                     -> Result<STResult>
-        where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>,
-              V: NetworkView {
+    fn process_message<V>(&mut self, view: V,
+                          message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
+                          -> Result<STResult> where V: NetworkView {
         let (header, message) = message.into_inner();
 
         let message = message.into_inner();
@@ -361,11 +349,8 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         Ok(STResult::StateTransferRunning)
     }
 
-    fn handle_app_state_requested<D, OP, LP, V>(&mut self, view: V, seq: SeqNo) -> Result<ExecutionResult>
-        where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>, V: NetworkView {
+    fn handle_app_state_requested<V>(&mut self, view: V, seq: SeqNo) -> Result<ExecutionResult>
+        where V: NetworkView {
         let earlier = std::mem::replace(&mut self.current_checkpoint_state, CheckpointState::None);
 
         self.current_checkpoint_state = match earlier {
@@ -388,12 +373,8 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         Ok(ExecutionResult::BeginCheckpoint)
     }
 
-    fn handle_timeout<D, OP, LP, V>(&mut self, view: V, timeout: Vec<RqTimeout>) -> Result<STTimeoutResult>
-        where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>,
-              V: NetworkView {
+    fn handle_timeout<V>(&mut self, view: V, timeout: Vec<RqTimeout>) -> Result<STTimeoutResult>
+        where V: NetworkView {
         for cst_seq in timeout {
             if let TimeoutKind::Cst(cst_seq) = cst_seq.timeout_kind() {
                 if self.cst_request_timed_out(cst_seq.clone(), view.clone()) {
@@ -408,7 +389,8 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
 
 impl<S, NT, PL> MonolithicStateTransfer<S, NT, PL> for CollabStateTransfer<S, NT, PL>
     where S: MonolithicState + 'static,
-          PL: MonolithicStateLog<S> + 'static {
+          PL: MonolithicStateLog<S> + 'static,
+          NT: StateTransferSendNode<CSTMsg<S>> + 'static {
     type Config = StateTransferConfig;
 
     fn initialize(config: Self::Config, timeouts: Timeouts, node: Arc<NT>,
@@ -422,12 +404,8 @@ impl<S, NT, PL> MonolithicStateTransfer<S, NT, PL> for CollabStateTransfer<S, NT
         Ok(Self::new(node, timeout_duration, timeouts, log, executor_handle))
     }
 
-    fn handle_state_received_from_app<D, OP, LP, V>(&mut self, view: V, state: Arc<ReadOnly<Checkpoint<S>>>) -> Result<()>
-        where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Self::Serialization, LP>>,
-              V: NetworkView {
+    fn handle_state_received_from_app<V>(&mut self, view: V, state: Arc<ReadOnly<Checkpoint<S>>>) -> Result<()>
+        where V: NetworkView {
         self.finalize_checkpoint(state)?;
 
         if self.needs_checkpoint() {
@@ -449,6 +427,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     where
         S: MonolithicState + 'static,
         PL: MonolithicStateLog<S> + 'static,
+        NT: StateTransferSendNode<CSTMsg<S>> + 'static
 {
     /// Create a new instance of `CollabStateTransfer`.
     pub fn new(node: Arc<NT>, base_timeout: Duration, timeouts: Timeouts, persistent_log: PL, install_channel: ChannelSyncTx<InstallStateMessage<S>>) -> Self {
@@ -477,15 +456,11 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         matches!(self.phase, ProtoPhase::WaitingCheckpoint(_))
     }
 
-    fn process_request_seq<D, OP, LP>(
+    fn process_request_seq<>(
         &mut self,
         header: Header,
         message: CstMessage<S>)
-        where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>
-    {
+        where {
         let seq = match &self.current_checkpoint_state {
             CheckpointState::PartialWithEarlier { seq, earlier, } => {
                 Some((earlier.sequence_number(), earlier.digest().clone()))
@@ -502,22 +477,17 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
         let reply = CstMessage::new(message.sequence_number(), kind);
 
-        let network_msg = SystemMessage::from_state_transfer_message(reply);
-
         debug!("{:?} // Replying to {:?} seq {:?} with seq no {:?}", self.node.id(),
             header.from(), message.sequence_number(), seq);
 
-        self.node.send(network_msg, header.from(), true);
+        self.node.send(reply, header.from(), true);
     }
 
 
     /// Process the entire list of pending state transfer requests
     /// This will only reply to the latest request sent by each of the replicas
-    fn process_pending_state_requests<D, OP, LP>(&mut self)
-        where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>> {
+    fn process_pending_state_requests(&mut self)
+        where {
         let waiting = std::mem::replace(&mut self.phase, ProtoPhase::Init);
 
         if let ProtoPhase::WaitingCheckpoint(reqs) = waiting {
@@ -542,19 +512,16 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
             map.into_values().for_each(|req| {
                 let (header, message) = req.into_inner();
 
-                self.process_request_state::<D, OP, LP>(header, message);
+                self.process_request_state(header, message);
             });
         }
     }
 
-    fn process_request_state<D, OP, LP>(
+    fn process_request_state(
         &mut self,
         header: Header,
         message: CstMessage<S>,
-    ) where D: ApplicationData + 'static,
-            OP: OrderingProtocolMessage + 'static,
-            LP: LogTransferMessage + 'static,
-            NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>
+    ) where
     {
         match &mut self.phase {
             ProtoPhase::Init => {}
@@ -592,22 +559,16 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
             }),
         );
 
-        let network_msg = SystemMessage::from_state_transfer_message(reply);
-
-        self.node.send(network_msg, header.from(), true).unwrap();
+        self.node.send(reply, header.from(), true).unwrap();
     }
 
     /// Advances the state of the CST state machine.
-    pub fn process_message<D, OP, LP, V>(
+    pub fn process_message<V>(
         &mut self,
         view: V,
         progress: CstProgress<S>,
     ) -> CstStatus<S>
-        where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
-              V: NetworkView
+        where V: NetworkView
     {
         match self.phase {
             ProtoPhase::WaitingCheckpoint(_) => {
@@ -868,11 +829,8 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
     /// Handle a timeout received from the timeouts layer.
     /// Returns a bool to signify if we must move to the Retrieving state
     /// If the timeout is no longer relevant, returns false (Can remain in current phase)
-    pub fn cst_request_timed_out<D, OP, LP, V>(&mut self, seq: SeqNo, view: V) -> bool
-        where D: ApplicationData + 'static,
-              OP: OrderingProtocolMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>, V: NetworkView {
+    pub fn cst_request_timed_out<V>(&mut self, seq: SeqNo, view: V) -> bool
+        where V: NetworkView {
         let status = self.timed_out(seq);
 
         match status {
@@ -923,14 +881,10 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
     /// Used by a recovering node to retrieve the latest sequence number
     /// attributed to a client request by the consensus layer.
-    pub fn request_latest_consensus_seq_no<D, OP, LP, V>(
+    pub fn request_latest_consensus_seq_no<V>(
         &mut self,
         view: V,
-    ) where D: ApplicationData + 'static,
-            OP: OrderingProtocolMessage + 'static,
-            LP: LogTransferMessage + 'static,
-            NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
-            V: NetworkView
+    ) where V: NetworkView
     {
 
         // reset state of latest seq no. request
@@ -956,17 +910,13 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
         let targets = view.quorum_members().clone().into_iter().filter(|id| *id != self.node.id());
 
-        self.node.broadcast(SystemMessage::from_state_transfer_message(message), targets);
+        self.node.broadcast(message, targets);
     }
 
     /// Used by a recovering node to retrieve the latest state.
-    pub fn request_latest_state<D, OP, LP, V>(
+    pub fn request_latest_state<V>(
         &mut self, view: V,
-    ) where D: ApplicationData + 'static,
-            OP: OrderingProtocolMessage + 'static,
-            LP: LogTransferMessage + 'static,
-            NT: ProtocolNetworkNode<ServiceMsg<D, OP, Ser<Self, S, NT, PL>, LP>>,
-            V: NetworkView
+    ) where V: NetworkView
     {
         // reset hashmap of received states
         self.received_states.clear();
@@ -988,7 +938,7 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         let message = CstMessage::new(cst_seq, CstMessageKind::RequestState);
         let targets = view.quorum_members().clone().into_iter().filter(|id| *id != self.node.id());
 
-        self.node.broadcast(SystemMessage::from_state_transfer_message(message), targets);
+        self.node.broadcast(message, targets);
     }
 }
 

From fa277391aa182f26e432c285bf7f615c40a9da47 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 24 Jul 2023 20:08:39 +0100
Subject: [PATCH 24/80] Created a new trait for executor replying nodes.

---
 febft-pbft-consensus/src/bft/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 2f8e40d6..bd4207b4 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -822,7 +822,7 @@ impl<D, ST, LP, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTC
     }
 }
 
-impl<D, ST, LP, NT, PL, RP> ReconfigurableOrderProtocol<RP, NT> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
+impl<D, ST, LP, NT, PL, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
     where D: ApplicationData + 'static,
           ST: StateTransferMessage + 'static,
           LP: LogTransferMessage + 'static,

From 0d483e7244012c972e5110541502c25cc2d33506 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Fri, 28 Jul 2023 15:22:31 +0100
Subject: [PATCH 25/80] Studying some really weird behaviour with extract_if
 used in the timeouts... Seems to not be properly extracting the requests.

---
 febft-state-transfer/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index a3c493c7..82cab562 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -305,7 +305,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
             _ => {}
         }
 
-// Notify timeouts that we have received this message
+        // Notify timeouts that we have received this message
         self.timeouts.received_cst_request(header.from(), message.sequence_number());
 
         let status = self.process_message(view.clone(),

From 218c995411e8ff3faead64e2f4162b03d889d538 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Sat, 29 Jul 2023 18:25:06 +0100
Subject: [PATCH 26/80] Moved order protocol to the new divided network so it
 no longer requires knowledge about the other protocols.

---
 febft-pbft-consensus/src/bft/config/mod.rs    |   7 +-
 .../src/bft/consensus/accessory/mod.rs        |  40 ++---
 .../bft/consensus/accessory/replica/mod.rs    |  59 +++----
 .../src/bft/consensus/decision/mod.rs         |  21 +--
 febft-pbft-consensus/src/bft/consensus/mod.rs |  33 ++--
 febft-pbft-consensus/src/bft/mod.rs           | 154 ++++++++----------
 febft-pbft-consensus/src/bft/proposer/mod.rs  |  31 ++--
 febft-pbft-consensus/src/bft/sync/mod.rs      | 113 ++++++-------
 .../src/bft/sync/replica_sync/mod.rs          |  35 ++--
 9 files changed, 209 insertions(+), 284 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/config/mod.rs b/febft-pbft-consensus/src/bft/config/mod.rs
index dbf4aaa7..5dc35247 100644
--- a/febft-pbft-consensus/src/bft/config/mod.rs
+++ b/febft-pbft-consensus/src/bft/config/mod.rs
@@ -9,7 +9,7 @@ use atlas_execution::serialize::ApplicationData;
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::sync::view::ViewInfo;
 
-pub struct PBFTConfig<D, ST, RP>
+pub struct PBFTConfig<D, RP>
     where D: ApplicationData, RP: ReconfigurationProtocolMessage + 'static {
     pub node_id: NodeId,
     // pub observer_handle: ObserverHandle,
@@ -18,12 +18,10 @@ pub struct PBFTConfig<D, ST, RP>
     pub timeout_dur: Duration,
     pub proposer_config: ProposerConfig,
     pub watermark: u32,
-    pub _phantom_data: PhantomData<ST>,
 }
 
 impl<D: ApplicationData + 'static,
-    ST: StateTransferMessage + 'static,
-    RP: ReconfigurationProtocolMessage + 'static> PBFTConfig<D, ST, RP> {
+    RP: ReconfigurationProtocolMessage + 'static> PBFTConfig<D, RP> {
     pub fn new(node_id: NodeId,
                follower_handle: Option<FollowerHandle<PBFTConsensus<D, RP>>>,
                view: ViewInfo, timeout_dur: Duration,
@@ -36,7 +34,6 @@ impl<D: ApplicationData + 'static,
             timeout_dur,
             proposer_config,
             watermark,
-            _phantom_data: Default::default(),
         }
     }
 }
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
index c9605169..507c08c7 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
@@ -1,9 +1,11 @@
 use std::sync::Arc;
 use atlas_communication::protocol_node::ProtocolNetworkNode;
+use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_execution::serialize::ApplicationData;
 
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
+use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::msg_log::deciding_log::DecidingLog;
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
 use crate::bft::PBFT;
@@ -11,68 +13,62 @@ use crate::bft::sync::view::ViewInfo;
 
 pub mod replica;
 
-pub enum ConsensusDecisionAccessory<D, ST, LP, RP>
+pub enum ConsensusDecisionAccessory<D, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static {
     Follower,
-    Replica(ReplicaAccessory<D, ST, LP, RP>),
+    Replica(ReplicaAccessory<D, RP>),
 }
 
-pub trait AccessoryConsensus<D, ST, LP, RP> where D: ApplicationData + 'static,
-                                                  ST: StateTransferMessage + 'static,
-                                                  LP: LogTransferMessage + 'static,
+pub trait AccessoryConsensus<D,RP> where D: ApplicationData + 'static,
                                                   RP: ReconfigurationProtocolMessage + 'static {
     /// Handle the reception of a pre-prepare message without having completed the pre prepare phase
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>;
+                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D,RP>> + 'static;
 
     /// Handle the prepare phase having been completed
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static;
+                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D,RP>> + 'static;
 
     /// Handle a prepare message processed during the preparing phase without having
     /// reached a quorum
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>;
+                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static;
 
     /// Handle a prepare message processed during the prepare phase when a quorum
     /// has been achieved
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
-                                   node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>;
+                                   node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static;
 
     /// Handle a commit message processed during the preparing phase without having
     /// reached a quorum
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
-                                       node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>;
+                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static;
 
     /// Handle a commit message processed during the prepare phase when a quorum has been reached
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
-                                    node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>;
+                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static;
 }
 
-impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ConsensusDecisionAccessory<D, ST, LP, RP>
+impl<D,  RP> AccessoryConsensus<D,  RP> for ConsensusDecisionAccessory<D, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static {
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -84,7 +80,7 @@ impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ConsensusDecisionAcces
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static {
+                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static{
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -96,7 +92,7 @@ impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ConsensusDecisionAcces
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -108,7 +104,7 @@ impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ConsensusDecisionAcces
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
-                                   node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+                                   node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -120,7 +116,7 @@ impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ConsensusDecisionAcces
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
-                                       node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -132,7 +128,7 @@ impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ConsensusDecisionAcces
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
-                                    node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index 13eb6e5d..457ae4a8 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -12,6 +12,7 @@ use atlas_communication::message::{NetworkMessageKind, SerializedMessage, Stored
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::messages::SystemMessage;
+use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, ServiceMsg, StateTransferMessage};
 use atlas_execution::serialize::ApplicationData;
 
@@ -20,31 +21,28 @@ use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::msg_log::deciding_log::DecidingLog;
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
 use crate::bft::{PBFT, SysMsg};
+use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::sync::view::ViewInfo;
 
-pub struct ReplicaAccessory<D, ST, LP, RP>
+pub struct ReplicaAccessory<D, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static {
-    speculative_commits: Arc<Mutex<BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP, RP>>>>>,
+    speculative_commits: Arc<Mutex<BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, RP>>>>>,
 }
 
-impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ReplicaAccessory<D, ST, LP, RP>
+impl<D, RP> AccessoryConsensus<D, RP> for ReplicaAccessory<D, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static {
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {}
+                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {}
 
     fn handle_pre_prepare_phase_completed<NT>(&mut self,
                                               deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               _msg: StoredConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static {
+                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static {
         let my_id = node.id();
         let view_seq = view.sequence_number();
         let current_digest = deciding_log.current_digest().unwrap();
@@ -59,12 +57,11 @@ impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ReplicaAccessory<D, ST
         let node_clone = node.clone();
 
         threadpool::execute(move || {
-            let message = SystemMessage::from_protocol_message(
-                PBFTMessage::Consensus(ConsensusMessage::new(
+            let message = PBFTMessage::Consensus(ConsensusMessage::new(
                     seq,
                     view_seq,
                     ConsensusMessageKind::Commit(current_digest.clone()),
-                )));
+                ));
 
             let (message, digest) = node_clone.serialize_digest_message(message).unwrap();
 
@@ -115,23 +112,23 @@ impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ReplicaAccessory<D, ST
 
         let targets = view.quorum_members().clone();
 
-        node.broadcast_signed(SystemMessage::from_protocol_message(message), targets.into_iter());
+        node.broadcast_signed(message, targets.into_iter());
     }
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {}
+                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {}
 
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
-                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
         let node_id = node.id();
 
         let seq = deciding_log.sequence_number();
         let current_digest = deciding_log.current_digest().unwrap();
         let speculative_commits = self.take_speculative_commits();
 
-        if valid_spec_commits::<D, ST, LP, RP>(&speculative_commits, node_id, seq, view) {
+        if valid_spec_commits::<D, RP>(&speculative_commits, node_id, seq, view) {
             for (_, msg) in speculative_commits.iter() {
                 debug!("{:?} // Broadcasting speculative commit message (total of {} messages) to {} targets",
                      node_id, speculative_commits.len(), view.params().n());
@@ -151,7 +148,7 @@ impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ReplicaAccessory<D, ST
 
             let targets = view.quorum_members().clone();
 
-            node.broadcast_signed(SystemMessage::from_protocol_message(message), targets.into_iter());
+            node.broadcast_signed(message, targets.into_iter());
         }
 
         debug!("{:?} // Broadcasted commit consensus message {:?}",
@@ -162,17 +159,15 @@ impl<D, ST, LP, RP> AccessoryConsensus<D, ST, LP, RP> for ReplicaAccessory<D, ST
 
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
-                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {}
+                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D,  RP>> {}
 
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
-                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {}
+                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D,  RP>> {}
 }
 
-impl<D, ST, LP, RP> ReplicaAccessory<D, ST, LP, RP>
+impl<D,  RP> ReplicaAccessory<D, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static {
     pub fn new() -> Self {
         Self {
@@ -180,7 +175,7 @@ impl<D, ST, LP, RP> ReplicaAccessory<D, ST, LP, RP>
         }
     }
 
-    fn take_speculative_commits(&self) -> BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP, RP>>> {
+    fn take_speculative_commits(&self) -> BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D,  RP>>> {
         let mut map = self.speculative_commits.lock().unwrap();
         std::mem::replace(&mut *map, BTreeMap::new())
     }
@@ -188,16 +183,14 @@ impl<D, ST, LP, RP> ReplicaAccessory<D, ST, LP, RP>
 
 
 #[inline]
-fn valid_spec_commits<D, ST, LP, RP>(
-    speculative_commits: &BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, ST, LP, RP>>>,
+fn valid_spec_commits<D,  RP>(
+    speculative_commits: &BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, RP>>>,
     node_id: NodeId,
     seq_no: SeqNo,
     view: &ViewInfo,
 ) -> bool
     where
         D: ApplicationData + 'static,
-        ST: StateTransferMessage + 'static,
-        LP: LogTransferMessage + 'static,
         RP: ReconfigurationProtocolMessage + 'static
 {
     let len = speculative_commits.len();
@@ -214,14 +207,6 @@ fn valid_spec_commits<D, ST, LP, RP>(
 
     speculative_commits
         .values()
-        .map(|stored| match stored.message().original() {
-            SystemMessage::ProtocolMessage(protocol) => {
-                match protocol.deref() {
-                    PBFTMessage::Consensus(consensus) => consensus,
-                    _ => { unreachable!() }
-                }
-            }
-            _ => { unreachable!() }
-        })
-        .all(|commit| commit.sequence_number() == seq_no)
+        .map(|msg| msg.message())
+        .all(|commit| commit.original().sequence_number() == seq_no)
 }
\ No newline at end of file
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 96ec4fa0..87edd3ec 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -13,6 +13,7 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::ClientRqInfo;
+use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog};
 use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
@@ -104,10 +105,8 @@ pub struct MessageQueue<O> {
 }
 
 /// The information needed to make a decision on a batch of requests.
-pub struct ConsensusDecision<D, ST, LP, PL, RP> where D: ApplicationData + 'static,
-                                                      ST: StateTransferMessage + 'static,
-                                                      LP: LogTransferMessage + 'static,
-                                                      RP: ReconfigurationProtocolMessage + 'static {
+pub struct ConsensusDecision<D, PL, RP> where D: ApplicationData + 'static,
+                                              RP: ReconfigurationProtocolMessage + 'static {
     node_id: NodeId,
     /// The sequence number of this consensus decision
     seq: SeqNo,
@@ -120,7 +119,7 @@ pub struct ConsensusDecision<D, ST, LP, PL, RP> where D: ApplicationData + 'stat
     /// Persistent log reference
     persistent_log: PL,
     /// Accessory to the base consensus state machine
-    accessory: ConsensusDecisionAccessory<D, ST, LP, RP>,
+    accessory: ConsensusDecisionAccessory<D, RP>,
     // Metrics about the consensus instance
     consensus_metrics: ConsensusMetrics,
     //TODO: Store things directly into the persistent log as well as delete them when
@@ -176,10 +175,8 @@ impl<O> MessageQueue<O> {
     }
 }
 
-impl<D, ST, LP, PL, RP> ConsensusDecision<D, ST, LP, PL, RP>
+impl<D, PL, RP> ConsensusDecision<D, PL, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static {
     pub fn init_decision(node_id: NodeId, seq_no: SeqNo, view: &ViewInfo, persistent_log: PL) -> Self {
         Self {
@@ -272,7 +269,7 @@ impl<D, ST, LP, PL, RP> ConsensusDecision<D, ST, LP, PL, RP>
                                timeouts: &Timeouts,
                                log: &mut Log<D, PL>,
                                node: &Arc<NT>) -> Result<DecisionStatus>
-        where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
               PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         let view = synchronizer.view();
 
@@ -526,7 +523,7 @@ impl<D, ST, LP, PL, RP> ConsensusDecision<D, ST, LP, PL, RP>
                     self.consensus_metrics.first_commit_recvd();
                 }
 
-                let stored_msg = Arc::new(ReadOnly::new(StoredMessage::new(header,message)));
+                let stored_msg = Arc::new(ReadOnly::new(StoredMessage::new(header, message)));
 
                 self.message_log.process_message(stored_msg.clone())?;
 
@@ -597,10 +594,8 @@ impl<D, ST, LP, PL, RP> ConsensusDecision<D, ST, LP, PL, RP>
     }
 }
 
-impl<D, ST, LP, PL, RP> Orderable for ConsensusDecision<D, ST, LP, PL, RP>
+impl<D, PL, RP> Orderable for ConsensusDecision<D, PL, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static {
     fn sequence_number(&self) -> SeqNo {
         self.seq
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 1cc5ae04..5a0518d6 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -13,6 +13,7 @@ use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_ad
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage, SystemMessage};
+use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
@@ -184,10 +185,8 @@ pub struct Signals {
 
 /// The consensus handler. Responsible for multiplexing consensus instances and keeping track
 /// of missing messages
-pub struct Consensus<D, ST, LP, PL, RP>
+pub struct Consensus<D, PL, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
           PL: Clone {
     node_id: NodeId,
@@ -204,7 +203,7 @@ pub struct Consensus<D, ST, LP, PL, RP>
     /// The consensus instances that are currently being processed
     /// A given consensus instance n will only be finished when all consensus instances
     /// j, where j < n have already been processed, in order to maintain total ordering
-    decisions: VecDeque<ConsensusDecision<D, ST, LP, PL, RP>>,
+    decisions: VecDeque<ConsensusDecision<D, PL, RP>>,
     /// The queue for messages that sit outside the range seq_no + watermark
     /// These messages cannot currently be processed since they sit outside the allowed
     /// zone but they will be processed once the seq no moves forward enough to include them
@@ -223,11 +222,9 @@ pub struct Consensus<D, ST, LP, PL, RP>
     persistent_log: PL,
 }
 
-impl<D, ST, LP, PL, RP> Consensus<D, ST, LP, PL, RP> where D: ApplicationData + 'static,
-                                                           ST: StateTransferMessage + 'static,
-                                                           LP: LogTransferMessage + 'static,
-                                                           RP: ReconfigurationProtocolMessage + 'static,
-                                                           PL: Clone {
+impl<D, PL, RP> Consensus<D, PL, RP> where D: ApplicationData + 'static,
+                                           RP: ReconfigurationProtocolMessage + 'static,
+                                           PL: Clone {
     pub fn new_replica(node_id: NodeId, view: &ViewInfo, executor_handle: ExecutorHandle<D>, seq_no: SeqNo,
                        watermark: u32, consensus_guard: Arc<ProposerConsensusGuard>, timeouts: Timeouts,
                        persistent_log: PL) -> Self {
@@ -372,7 +369,7 @@ impl<D, ST, LP, PL, RP> Consensus<D, ST, LP, PL, RP> where D: ApplicationData +
                                timeouts: &Timeouts,
                                log: &mut Log<D, PL>,
                                node: &Arc<NT>) -> Result<ConsensusStatus>
-        where NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
               PL: OrderingProtocolLog<PBFTConsensus<D, RP>>, {
         let message_seq = message.sequence_number();
 
@@ -485,7 +482,7 @@ impl<D, ST, LP, PL, RP> Consensus<D, ST, LP, PL, RP> where D: ApplicationData +
     /// Advance to the next instance of the consensus
     /// This will also create the necessary new decision to keep the pending decisions
     /// equal to the water mark
-    pub fn next_instance(&mut self, view: &ViewInfo) -> ConsensusDecision<D, ST, LP, PL, RP> {
+    pub fn next_instance(&mut self, view: &ViewInfo) -> ConsensusDecision<D, PL, RP> {
         let decision = self.decisions.pop_front().unwrap();
 
         self.seq_no = self.seq_no.next();
@@ -721,15 +718,15 @@ impl<D, ST, LP, PL, RP> Consensus<D, ST, LP, PL, RP> where D: ApplicationData +
         &self,
         requests: Vec<StoredRequestMessage<D::Request>>,
         synchronizer: &K,
-    ) -> SysMsg<D, ST, LP, RP>
+    ) -> SysMsg<D, RP>
         where
             K: AbstractSynchronizer<D, RP>,
     {
-        SystemMessage::from_protocol_message(PBFTMessage::Consensus(ConsensusMessage::new(
+        PBFTMessage::Consensus(ConsensusMessage::new(
             self.sequence_number(),
             synchronizer.view().sequence_number(),
             ConsensusMessageKind::PrePrepare(requests),
-        )))
+        ))
     }
 
     /// Install a given view into the current consensus decisions.
@@ -809,7 +806,7 @@ impl<D, ST, LP, PL, RP> Consensus<D, ST, LP, PL, RP> where D: ApplicationData +
         log: &mut Log<D, PL>,
         node: &Arc<NT>,
     ) where
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+        NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
         PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         let view = synchronizer.view();
         //Prepare the algorithm as we are already entering this phase
@@ -852,7 +849,7 @@ impl<D, ST, LP, PL, RP> Consensus<D, ST, LP, PL, RP> where D: ApplicationData +
     }
 
     /// Enqueue a decision onto our overlapping decision log
-    fn enqueue_decision(&mut self, decision: ConsensusDecision<D, ST, LP, PL, RP>) {
+    fn enqueue_decision(&mut self, decision: ConsensusDecision<D, PL, RP>) {
         self.signalled.push_signalled(decision.sequence_number());
 
         self.decisions.push_back(decision);
@@ -874,10 +871,8 @@ impl<D, ST, LP, PL, RP> Consensus<D, ST, LP, PL, RP> where D: ApplicationData +
     }
 }
 
-impl<D, ST, LP, PL, RP> Orderable for Consensus<D, ST, LP, PL, RP>
+impl<D, PL, RP> Orderable for Consensus<D, PL, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
           PL: Clone {
     fn sequence_number(&self) -> SeqNo {
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index bd4207b4..c0e5562b 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -27,12 +27,13 @@ use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::messages::Protocol;
 use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, ProtocolConsensusDecision, ProtocolMessage, SerProof, SerProofMetadata, View};
+use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::{ReconfigurableOrderProtocol, ReconfigurationAttemptResult};
 use atlas_core::ordering_protocol::stateful_order_protocol::{DecLog, StatefulOrderProtocol};
 use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol, StatefulOrderingProtocolLog};
 use atlas_core::reconfiguration_protocol::{QuorumJoinCert, ReconfigurationProtocol};
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
-use atlas_core::serialize::{LogTransferMessage, NetworkView, ReconfigurationProtocolMessage, ServiceMsg, StateTransferMessage};
+use atlas_core::serialize::{LogTransferMessage, NetworkView, OrderingProtocolMessage, ReconfigurationProtocolMessage, ServiceMsg, StateTransferMessage};
 use atlas_core::state_transfer::{Checkpoint};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_metrics::metrics::metric_duration;
@@ -57,9 +58,9 @@ pub mod observer;
 pub mod metric;
 
 // The types responsible for this protocol
-pub type PBFT<D, ST, LP, RP> = ServiceMsg<D, PBFTConsensus<D, RP>, ST, LP>;
+pub type PBFT<D, RP> = PBFTConsensus<D, RP>;
 // The message type for this consensus protocol
-pub type SysMsg<D, ST, LP, RP> = <PBFT<D, ST, LP, RP> as Serializable>::Message;
+pub type SysMsg<D, RP> = <PBFTConsensus<D, RP> as OrderingProtocolMessage>::ProtocolMessage;
 
 #[derive(Clone, PartialEq, Eq, Debug)]
 /// Which phase of the consensus algorithm are we currently executing
@@ -80,18 +81,16 @@ pub enum SyncPhaseRes<O> {
 }
 
 /// a PBFT based ordering protocol
-pub struct PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
+pub struct PBFTOrderProtocol<D, NT, PL, RP>
     where
         D: ApplicationData + 'static,
-        ST: StateTransferMessage + 'static,
-        LP: LogTransferMessage + 'static,
         RP: ReconfigurationProtocolMessage + 'static,
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+        NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
         PL: Clone {
     // What phase of the consensus algorithm are we currently executing
     phase: ConsensusPhase,
     /// The consensus state machine
-    consensus: Consensus<D, ST, LP, PL, RP>,
+    consensus: Consensus<D, PL, RP>,
     /// The synchronizer state machine
     synchronizer: Arc<Synchronizer<D, RP>>,
     /// The request pre processor
@@ -115,34 +114,35 @@ pub struct PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
     executor: ExecutorHandle<D>,
 }
 
-impl<D, ST, LP, NT, PL, RP> Orderable for PBFTOrderProtocol<D, ST, LP, NT, PL, RP> where D: 'static + ApplicationData,
-                                                                                         NT: 'static + ProtocolNetworkNode<PBFT<D, ST, LP, RP>>,
-                                                                                         ST: 'static + StateTransferMessage,
-                                                                                         LP: 'static + LogTransferMessage,
-                                                                                         RP: 'static + ReconfigurationProtocolMessage,
-                                                                                         PL: Clone {
+impl<D, NT, PL, RP> Orderable for PBFTOrderProtocol<D, NT, PL, RP>
+    where D: 'static + ApplicationData,
+          NT: 'static + OrderProtocolSendNode<D, PBFT<D, RP>>,
+          RP: 'static + ReconfigurationProtocolMessage,
+          PL: Clone {
     fn sequence_number(&self) -> SeqNo {
         self.consensus.sequence_number()
     }
 }
 
-impl<D, ST, LP, NT, PL, RP> OrderProtocolTolerance for PBFTOrderProtocol<D, ST, LP, NT, PL, RP> where D: 'static + ApplicationData, LP: 'static + LogTransferMessage, NT: 'static + ProtocolNetworkNode<PBFT<D, ST, LP, RP>>, PL: Clone, RP: 'static + ReconfigurationProtocolMessage, ST: 'static + StateTransferMessage {
+impl<D, NT, PL, RP> OrderProtocolTolerance for PBFTOrderProtocol<D, NT, PL, RP>
+    where D: 'static + ApplicationData,
+          NT: 'static + OrderProtocolSendNode<D, PBFT<D, RP>>,
+          PL: Clone,
+          RP: 'static + ReconfigurationProtocolMessage {
     fn get_n_for_f(f: usize) -> usize {
         3 * f + 1
     }
 }
 
-impl<D, ST, LP, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
+impl<D, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
           PL: Clone {
     type Serialization = PBFTConsensus<D, RP>;
-    type Config = PBFTConfig<D, ST, RP>;
+    type Config = PBFTConfig<D, RP>;
 
-    fn initialize(config: PBFTConfig<D, ST, RP>, args: OrderingProtocolArgs<D, NT, PL>) -> Result<Self> where
+    fn initialize(config: PBFTConfig<D, RP>, args: OrderingProtocolArgs<D, NT, PL>) -> Result<Self> where
         Self: Sized,
     {
         Self::initialize_protocol(config, args, None)
@@ -171,6 +171,23 @@ impl<D, ST, LP, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D,
         }
     }
 
+    fn handle_execution_changed(&mut self, is_executing: bool) -> Result<()> {
+        if !is_executing {
+            self.consensus_guard.lock_consensus();
+        } else {
+            match self.phase {
+                ConsensusPhase::NormalPhase => {
+                    self.consensus_guard.unlock_consensus();
+                }
+                ConsensusPhase::SyncPhase => {}
+            }
+        }
+
+        info!("Execution has changed to {} with our current phase being {:?}", is_executing, self.phase);
+
+        Ok(())
+    }
+
     fn poll(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request, QuorumJoinCert<RP>>, D::Request>
         where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         match self.phase {
@@ -195,6 +212,25 @@ impl<D, ST, LP, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D,
         }
     }
 
+    fn sequence_number_with_proof(&self) -> Result<Option<(SeqNo, SerProof<Self::Serialization>)>> {
+        Ok(self.message_log.last_proof(self.synchronizer.view().f())
+            .map(|p| (p.sequence_number(), p)))
+    }
+
+    fn verify_sequence_number(&self, seq_no: SeqNo, proof: &SerProof<Self::Serialization>) -> Result<bool> {
+        let proof: &Proof<D::Request> = proof;
+
+        //TODO: Verify the proof
+
+        Ok(true)
+    }
+
+    fn install_seq_no(&mut self, seq_no: SeqNo) -> Result<()> {
+        self.consensus.install_sequence_number(seq_no, &self.synchronizer.view());
+
+        Ok(())
+    }
+
     fn handle_timeout(&mut self, timeout: Vec<RqTimeout>) -> Result<OrderProtocolExecResult<D::Request>>
         where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         if self.consensus.is_catching_up() {
@@ -234,58 +270,20 @@ impl<D, ST, LP, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D,
 
         Ok(OrderProtocolExecResult::Success)
     }
-
-    fn sequence_number_with_proof(&self) -> Result<Option<(SeqNo, SerProof<Self::Serialization>)>> {
-        Ok(self.message_log.last_proof(self.synchronizer.view().f())
-            .map(|p| (p.sequence_number(), p)))
-    }
-
-    fn verify_sequence_number(&self, seq_no: SeqNo, proof: &SerProof<Self::Serialization>) -> Result<bool> {
-        let proof: &Proof<D::Request> = proof;
-
-        //TODO: Verify the proof
-
-        Ok(true)
-    }
-
-    fn handle_execution_changed(&mut self, is_executing: bool) -> Result<()> {
-        if !is_executing {
-            self.consensus_guard.lock_consensus();
-        } else {
-            match self.phase {
-                ConsensusPhase::NormalPhase => {
-                    self.consensus_guard.unlock_consensus();
-                }
-                ConsensusPhase::SyncPhase => {}
-            }
-        }
-
-        info!("Execution has changed to {} with our current phase being {:?}", is_executing, self.phase);
-
-        Ok(())
-    }
-
-    fn install_seq_no(&mut self, seq_no: SeqNo) -> Result<()> {
-        self.consensus.install_sequence_number(seq_no, &self.synchronizer.view());
-
-        Ok(())
-    }
 }
 
-impl<D, ST, LP, NT, PL, RP> PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
+impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
           PL: Clone {
-    fn initialize_protocol(config: PBFTConfig<D, ST, RP>, args: OrderingProtocolArgs<D, NT, PL>,
+    fn initialize_protocol(config: PBFTConfig<D, RP>, args: OrderingProtocolArgs<D, NT, PL>,
                            initial_state: Option<DecisionLog<D::Request>>) -> Result<Self> {
         let PBFTConfig {
             node_id,
             follower_handle,
             view, timeout_dur,
-            proposer_config, watermark, _phantom_data
+            proposer_config, watermark
         } = config;
 
         let OrderingProtocolArgs(executor, timeouts,
@@ -296,9 +294,9 @@ impl<D, ST, LP, NT, PL, RP> PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
 
         let consensus_guard = ProposerConsensusGuard::new(view.clone(), watermark);
 
-        let consensus = Consensus::<D, ST, LP, PL, RP>::new_replica(node_id, &sync.view(), executor.clone(),
-                                                                    SeqNo::ZERO, watermark, consensus_guard.clone(),
-                                                                    timeouts.clone(), persistent_log.clone());
+        let consensus = Consensus::<D, PL, RP>::new_replica(node_id, &sync.view(), executor.clone(),
+                                                            SeqNo::ZERO, watermark, consensus_guard.clone(),
+                                                            timeouts.clone(), persistent_log.clone());
 
         let dec_log = initialize_decided_log::<D, PL>(node_id, persistent_log, initial_state)?;
 
@@ -615,12 +613,10 @@ impl<D, ST, LP, NT, PL, RP> PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
     }
 }
 
-impl<D, ST, LP, NT, PL, RP> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
+impl<D, NT, PL, RP> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
           PL: Clone {
     type StateSerialization = PBFTConsensus<D, RP>;
 
@@ -680,12 +676,10 @@ impl<D, ST, LP, NT, PL, RP> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtoc
     }
 }
 
-impl<D, ST, LP, NT, PL, RP> PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
+impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
           PL: Clone {
     pub(crate) fn switch_phase(&mut self, new_phase: ConsensusPhase) {
         info!("{:?} // Switching from phase {:?} to phase {:?}", self.node.id(), self.phase, new_phase);
@@ -745,12 +739,10 @@ const CF_PRE_PREPARES: &str = "PRE_PREPARES";
 const CF_PREPARES: &str = "PREPARES";
 const CF_COMMIT: &str = "COMMITS";
 
-impl<D, ST, LP, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTConsensus<D, RP>> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
+impl<D, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTConsensus<D, RP>> for PBFTOrderProtocol<D, NT, PL, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>, PL: Clone {
+          NT: OrderProtocolSendNode<D, PBFT<D, RP>>, PL: Clone {
     fn message_types() -> Vec<&'static str> {
         vec![
             CF_PRE_PREPARES,
@@ -822,12 +814,10 @@ impl<D, ST, LP, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTC
     }
 }
 
-impl<D, ST, LP, NT, PL, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, ST, LP, NT, PL, RP>
+impl<D, NT, PL, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, NT, PL, RP>
     where D: ApplicationData + 'static,
-          ST: StateTransferMessage + 'static,
-          LP: LogTransferMessage + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
-          NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
           PL: Clone {
     fn attempt_network_view_change(&mut self, join_certificate: QuorumJoinCert<RP>) -> Result<ReconfigurationAttemptResult> {
         Ok(self.synchronizer.attempt_join_quorum(join_certificate, &*self.node, &self.timeouts))
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index eb2dc609..f5e9d305 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -12,7 +12,7 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::threadpool;
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
-use atlas_core::reconfiguration_protocol::ReconfigurationProtocol;
+use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::request_pre_processing::{BatchOutput, PreProcessorOutputMessage};
 use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
@@ -109,10 +109,8 @@ impl<D, NT, RP> Proposer<D, NT, RP>
     }
 
     ///Start this work
-    pub fn start<ST, LP>(self: Arc<Self>) -> JoinHandle<()>
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static {
+    pub fn start(self: Arc<Self>) -> JoinHandle<()>
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static {
         std::thread::Builder::new()
             .name(format!("Proposer thread"))
             .spawn(move || {
@@ -247,12 +245,10 @@ impl<D, NT, RP> Proposer<D, NT, RP>
     /// Attempt to propose an unordered request batch
     /// Fails if the batch is not large enough or the timeout
     /// Has not yet occurred
-    fn propose_unordered<ST, LP>(
+    fn propose_unordered(
         &self,
         propose: &mut ProposeBuilder<D>,
-    ) -> bool where ST: StateTransferMessage + 'static,
-                    LP: LogTransferMessage + 'static,
-                    NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+    ) -> bool where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
         if !propose.currently_accumulated.is_empty() {
             let current_batch_size = propose.currently_accumulated.len();
 
@@ -318,12 +314,9 @@ impl<D, NT, RP> Proposer<D, NT, RP>
 
     /// attempt to propose the ordered requests that we have collected
     /// Returns true if a batch was proposed
-    fn propose_ordered<ST, LP>(&self,
-                               is_leader: bool,
-                               propose: &mut ProposeBuilder<D>, ) -> bool
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+    fn propose_ordered(&self, is_leader: bool,
+                       propose: &mut ProposeBuilder<D>, ) -> bool
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
 
         //Now let's deal with ordered requests
         if is_leader {
@@ -375,15 +368,13 @@ impl<D, NT, RP> Proposer<D, NT, RP>
 
     /// Proposes a new batch.
     /// (Basically broadcasts it to all of the members)
-    fn propose<ST, LP>(
+    fn propose(
         &self,
         seq: SeqNo,
         view: &ViewInfo,
         mut currently_accumulated: Vec<StoredRequestMessage<D::Request>>,
     ) where
-        ST: StateTransferMessage + 'static,
-        LP: LogTransferMessage + 'static,
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+        NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
         let has_pending_messages = self.consensus_guard.has_pending_view_change_reqs();
 
         let is_view_change_empty = {
@@ -425,7 +416,7 @@ impl<D, NT, RP> Proposer<D, NT, RP>
 
         let targets = view.quorum_members().clone();
 
-        self.node_ref.broadcast_signed(SystemMessage::from_protocol_message(message), targets.into_iter());
+        self.node_ref.broadcast_signed(message, targets.into_iter());
 
         metric_increment(PROPOSER_BATCHES_MADE_ID, Some(1));
     }
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 89046e1f..7fef9f4e 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -21,9 +21,9 @@ use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_pop_message, tbo_queue_message};
 use atlas_communication::message::{Header, StoredMessage, WireMessage};
-use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
+use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::ReconfigurationAttemptResult;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
@@ -552,19 +552,17 @@ impl<D, RP> Synchronizer<D, RP>
     /// Advances the state of the view change state machine.
     //
     // TODO: retransmit STOP msgs
-    pub fn process_message<ST, LP, NT, PL>(
+    pub fn process_message<NT, PL>(
         &self,
         header: Header,
         message: ViewChangeMessage<D::Request, QuorumJoinCert<RP>>,
         timeouts: &Timeouts,
         log: &mut Log<D, PL>,
         rq_pre_processor: &RequestPreProcessor<D::Request>,
-        consensus: &mut Consensus<D, ST, LP, PL, RP>,
+        consensus: &mut Consensus<D, PL, RP>,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
               PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
         debug!("{:?} // Processing view change message {:?} in phase {:?} from {:?}",
@@ -1058,12 +1056,14 @@ impl<D, RP> Synchronizer<D, RP>
                                     Some(&*node_sign),
                                 ).into_inner();
 
-                                if let PBFTMessage::Consensus(consensus) = message.into_protocol_message() {
-                                    (h, consensus)
-                                } else {
-                                    //This is basically impossible
-                                    panic!("Returned random message from forge propose?")
+                                (h, message)
+                            };
+
+                            let message = match message {
+                                PBFTMessage::Consensus(msg) => {
+                                    msg
                                 }
+                                _ => unreachable!()
                             };
 
                             let fwd_request = FwdConsensusMessage::new(header, message);
@@ -1084,7 +1084,7 @@ impl<D, RP> Synchronizer<D, RP>
                             let targets = current_view.quorum_members().clone().into_iter()
                                 .filter(move |&id| id != node_id);
 
-                            node.broadcast(SystemMessage::from_protocol_message(message), targets);
+                            node.broadcast(message, targets);
 
                             let state = FinalizeState {
                                 curr_cid,
@@ -1182,9 +1182,9 @@ impl<D, RP> Synchronizer<D, RP>
 
                 // leader has already performed this computation in the
                 // STOP-DATA phase of Mod-SMaRt
-                let signed: Vec<_> = signed_collects::<D, _, _, _, _>(&**node, collects);
+                let signed: Vec<_> = signed_collects::<D, _, _>(&**node, collects);
 
-                let proof = highest_proof::<D, _, _, _, _, _>(&current_view, &**node, signed.iter());
+                let proof = highest_proof::<D, _, _, _>(&current_view, &**node, signed.iter());
 
                 let curr_cid = proof
                     .map(|p| p.sequence_number())
@@ -1232,16 +1232,14 @@ impl<D, RP> Synchronizer<D, RP>
     }
 
     /// Resume the view change protocol after running the CST protocol.
-    pub fn resume_view_change<ST, LP, NT, PL>(
+    pub fn resume_view_change<NT, PL>(
         &self,
         log: &mut Log<D, PL>,
         timeouts: &Timeouts,
-        consensus: &mut Consensus<D, ST, LP, PL, RP>,
+        consensus: &mut Consensus<D, PL, RP>,
         node: &Arc<NT>,
     ) -> Option<()>
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
               PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
         let state = self.finalize_state.borrow_mut().take()?;
@@ -1263,15 +1261,13 @@ impl<D, RP> Synchronizer<D, RP>
         Some(())
     }
 
-    pub fn begin_quorum_view_change<ST, LP, NT, PL>(
+    pub fn begin_quorum_view_change<NT, PL>(
         &self,
         join_cert: Option<QuorumJoinCert<RP>>,
         node: &NT,
         timeouts: &Timeouts,
         _log: &Log<D, PL>, )
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>,
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
         match (self.phase.get(), &join_cert) {
@@ -1294,12 +1290,10 @@ impl<D, RP> Synchronizer<D, RP>
         }
     }
 
-    pub fn attempt_join_quorum<ST, LP, NT>(&self, join_certificate: QuorumJoinCert<RP>,
-                                           node: &NT,
-                                           timeouts: &Timeouts) -> ReconfigurationAttemptResult
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>, {
+    pub fn attempt_join_quorum<NT>(&self, join_certificate: QuorumJoinCert<RP>,
+                                   node: &NT,
+                                   timeouts: &Timeouts) -> ReconfigurationAttemptResult
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>>, {
         let current_view = self.view();
 
         if current_view.quorum_members().contains(&node.id()) {
@@ -1313,7 +1307,7 @@ impl<D, RP> Synchronizer<D, RP>
             ViewChangeMessageKind::NodeQuorumJoin(node.id(), join_certificate),
         );
 
-        node.broadcast_signed(SystemMessage::from_protocol_message(PBFTMessage::ViewChange(message)), current_view.quorum_members().clone().into_iter());
+        node.broadcast_signed(PBFTMessage::ViewChange(message), current_view.quorum_members().clone().into_iter());
 
         // Simulate that we were accepted into the quorum
         let view = current_view.next_view_with_new_node(node.id());
@@ -1337,16 +1331,14 @@ impl<D, RP> Synchronizer<D, RP>
     /// that have timed out on the current replica.
     /// If the timed out requests are None, that means that the view change
     /// originated in the other replicas.
-    pub fn begin_view_change<ST, LP, NT, PL>(
+    pub fn begin_view_change<NT, PL>(
         &self,
         timed_out: Option<Vec<StoredRequestMessage<D::Request>>>,
         node: &NT,
         timeouts: &Timeouts,
         _log: &Log<D, PL>,
     )
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>,
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
         match (self.phase.get(), &timed_out) {
@@ -1438,18 +1430,17 @@ impl<D, RP> Synchronizer<D, RP>
 
     /// Finalize a view change and install the new view in the other
     /// state machines (Consensus)
-    fn finalize<ST, LP, NT, PL>(
+    fn finalize<NT, PL>(
         &self,
         state: FinalizeState<D::Request>,
         log: &mut Log<D, PL>,
         timeouts: &Timeouts,
-        consensus: &mut Consensus<D, ST, LP, PL, RP>,
+        consensus: &mut Consensus<D, PL, RP>,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
+        where
+            NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
+            PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
         let FinalizeState {
             curr_cid,
@@ -1533,12 +1524,10 @@ impl<D, RP> Synchronizer<D, RP>
     /// Forward the requests that have timed out to the whole network
     /// So that everyone knows about (including a leader that could still be correct, but
     /// Has not received the requests from the client)
-    pub fn forward_requests<ST, LP, NT>(&self,
-                                        timed_out: Vec<StoredRequestMessage<D::Request>>,
-                                        node: &NT)
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+    pub fn forward_requests<NT>(&self,
+                                timed_out: Vec<StoredRequestMessage<D::Request>>,
+                                node: &NT)
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
         match &self.accessory {
             SynchronizerAccessory::Follower(_) => {}
             SynchronizerAccessory::Replica(rep) => {
@@ -1583,16 +1572,14 @@ impl<D, RP> Synchronizer<D, RP>
 
     // TODO: quorum sizes may differ when we implement reconfiguration
     #[inline]
-    fn highest_proof<'a, ST, LP, NT>(
+    fn highest_proof<'a, NT>(
         guard: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
         view: &ViewInfo,
         node: &NT,
     ) -> Option<&'a Proof<D::Request>>
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>
+        where NT: OrderProtocolSendNode<D, PBFT<D, RP>>
     {
-        highest_proof::<D, _, _, _, _, _>(&view, node, guard.values())
+        highest_proof::<D, _, _, _>(&view, node, guard.values())
     }
 }
 
@@ -1817,30 +1804,26 @@ fn normalized_collects<'a, O: 'a>(
     })
 }
 
-fn signed_collects<D, ST, LP, NT, RP>(
+fn signed_collects<D, NT, RP>(
     node: &NT,
     collects: Vec<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
 ) -> Vec<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>
     where
         D: ApplicationData + 'static,
-        ST: StateTransferMessage + 'static,
-        LP: LogTransferMessage + 'static,
         RP: ReconfigurationProtocolMessage + 'static,
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>
+        NT: OrderProtocolSendNode<D, PBFT<D, RP>>
 {
     collects
         .into_iter()
-        .filter(|stored| validate_signature::<D, _, _, _, _, _>(node, stored))
+        .filter(|stored| validate_signature::<D, _, _, _>(node, stored))
         .collect()
 }
 
-fn validate_signature<'a, D, M, ST, LP, NT, RP>(node: &'a NT, stored: &'a StoredMessage<M>) -> bool
+fn validate_signature<'a, D, M, NT, RP>(node: &'a NT, stored: &'a StoredMessage<M>) -> bool
     where
         D: ApplicationData + 'static,
-        ST: StateTransferMessage + 'static,
-        LP: LogTransferMessage + 'static,
         RP: ReconfigurationProtocolMessage + 'static,
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>
+        NT: OrderProtocolSendNode<D, PBFT<D, RP>>
 {
     //TODO: Fix this as I believe it will always be false
     let wm = match WireMessage::from_header(*stored.header()) {
@@ -1866,7 +1849,7 @@ fn validate_signature<'a, D, M, ST, LP, NT, RP>(node: &'a NT, stored: &'a Stored
     wm.is_valid(Some(&key), false)
 }
 
-fn highest_proof<'a, D, I, ST, LP, NT, RP>(
+fn highest_proof<'a, D, I, NT, RP>(
     view: &ViewInfo,
     node: &NT,
     collects: I,
@@ -1875,9 +1858,7 @@ fn highest_proof<'a, D, I, ST, LP, NT, RP>(
         D: ApplicationData + 'static,
         I: Iterator<Item=&'a StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
         RP: ReconfigurationProtocolMessage + 'static,
-        ST: StateTransferMessage + 'static,
-        LP: LogTransferMessage + 'static,
-        NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>
+        NT: OrderProtocolSendNode<D, PBFT<D, RP>>
 {
     collect_data(collects)
         // fetch proofs
@@ -1897,7 +1878,7 @@ fn highest_proof<'a, D, I, ST, LP, NT, RP>(
                         .unwrap_or(false)
                 })
                 .filter(move |&stored|
-                    { validate_signature::<D, _, _, _, _, _>(node, stored) })
+                    { validate_signature::<D, _, _, _>(node, stored) })
                 .count() >= view.params().quorum();
 
             let prepares_valid = proof
@@ -1911,7 +1892,7 @@ fn highest_proof<'a, D, I, ST, LP, NT, RP>(
                         .unwrap_or(false)
                 })
                 .filter(move |&stored|
-                    { validate_signature::<D, _, _, _, _, _>(node, stored) })
+                    { validate_signature::<D, _, _, _>(node, stored) })
                 .count() >= view.params().quorum();
 
             debug!("{:?} // Proof {:?} is valid? commits valid: {:?} &&  prepares_valid: {:?}",
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 82b791da..fdd8a6c5 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -16,6 +16,7 @@ use atlas_communication::message::{NetworkMessageKind};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, RequestMessage, StoredRequestMessage, SystemMessage};
+use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
 use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
@@ -58,20 +59,18 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     ///
     /// Therefore, we start by clearing our stopped requests and treating them as
     /// newly proposed requests (by resetting their timer)
-    pub(super) fn handle_stopping_quorum<ST, LP, NT, PL, RP>(
+    pub(super) fn handle_stopping_quorum<NT, PL, RP>(
         &self,
         base_sync: &Synchronizer<D, RP>,
         previous_view: ViewInfo,
-        consensus: &Consensus<D, ST, LP, PL, RP>,
+        consensus: &Consensus<D, PL, RP>,
         log: &Log<D, PL>,
         pre_processor: &RequestPreProcessor<D::Request>,
         timeouts: &Timeouts,
         node: &NT,
     )
-        where ST: StateTransferMessage + 'static,
-              LP: LogTransferMessage + 'static,
-              RP: ReconfigurationProtocolMessage + 'static,
-              NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>>,
+        where RP: ReconfigurationProtocolMessage + 'static,
+              NT: OrderProtocolSendNode<D, PBFT<D, RP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
         // NOTE:
         // - install new view (i.e. update view seq no) (Done in the synchronizer)
@@ -105,22 +104,20 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
             ViewChangeMessageKind::StopData(collect),
         ));
 
-        node.send_signed(SystemMessage::from_protocol_message(message), current_leader, true);
+        node.send_signed(message, current_leader, true);
     }
 
     /// Start a new view change
     /// Receives the requests that it should send to the other
     /// nodes in its STOP message
-    pub(super) fn handle_begin_view_change<ST, LP, NT, RP>(
+    pub(super) fn handle_begin_view_change<NT, RP>(
         &self,
         base_sync: &Synchronizer<D, RP>,
         timeouts: &Timeouts,
         node: &NT,
         timed_out: Option<Vec<StoredRequestMessage<D::Request>>>,
-    ) where ST: StateTransferMessage + 'static,
-            LP: LogTransferMessage + 'static,
-            RP: ReconfigurationProtocolMessage + 'static,
-            NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
+    ) where RP: ReconfigurationProtocolMessage + 'static,
+            NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
         // stop all timers
         self.unwatch_all_requests(timeouts);
 
@@ -143,7 +140,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
 
         let targets = current_view.quorum_members().clone();
 
-        node.broadcast(SystemMessage::from_protocol_message(message), targets.into_iter());
+        node.broadcast(message, targets.into_iter());
     }
 
     /// Watch a vector of requests received
@@ -335,21 +332,19 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
 
     /// Forward the requests that timed out, `timed_out`, to all the nodes in the
     /// current view.
-    pub fn forward_requests<ST, LP, NT, RP>(
+    pub fn forward_requests<NT, RP>(
         &self,
         base_sync: &Synchronizer<D, RP>,
         timed_out: Vec<StoredRequestMessage<D::Request>>,
         node: &NT,
-    ) where ST: StateTransferMessage + 'static,
-            LP: LogTransferMessage + 'static,
-            RP: ReconfigurationProtocolMessage + 'static,
-            NT: ProtocolNetworkNode<PBFT<D, ST, LP, RP>> {
-        let message = SystemMessage::ForwardedRequestMessage(ForwardedRequestsMessage::new(timed_out));
+    ) where RP: ReconfigurationProtocolMessage + 'static,
+            NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
+        let message = ForwardedRequestsMessage::new(timed_out);
         let view = base_sync.view();
 
         let targets = view.quorum_members().clone();
 
-        node.broadcast(message, targets.into_iter());
+        node.forward_requests(message, targets.into_iter());
     }
 
     /// Obtain the requests that we know have timed out so we can send out a stop message

From 4f5480d796716f1c11b1fb6deee9fa58f00191d8 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 31 Jul 2023 18:53:46 +0100
Subject: [PATCH 27/80] Improving logging as I track down the bug where a newly
 joined replica stops working after some executions.

---
 .../src/bft/consensus/decision/mod.rs         | 12 ++--
 febft-pbft-consensus/src/bft/mod.rs           |  5 ++
 febft-pbft-consensus/src/bft/sync/mod.rs      | 61 +++++++++++++++----
 3 files changed, 60 insertions(+), 18 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 87edd3ec..887dfee3 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -288,24 +288,24 @@ impl<D, PL, RP> ConsensusDecision<D, PL, RP>
                     ConsensusMessageKind::PrePrepare(_)
                     if message.view() != view.sequence_number() => {
                         // drop proposed value in a different view (from different leader)
-                        debug!("{:?} // Dropped pre prepare message because of view {:?} vs {:?} (ours)",
-                            self.node_id, message.view(), synchronizer.view().sequence_number());
+                        debug!("{:?} // Dropped {:?} because of view {:?} vs {:?} (ours) header {:?}",
+                            self.node_id, message, message.view(), synchronizer.view().sequence_number(), header);
 
                         return Ok(DecisionStatus::Deciding);
                     }
                     ConsensusMessageKind::PrePrepare(_)
                     if !view.leader_set().contains(&header.from()) => {
                         // Drop proposed value since sender is not leader
-                        debug!("{:?} // Dropped pre prepare message because the sender was not the leader {:?} vs {:?} (ours)",
-                        self.node_id, header.from(), view.leader());
+                        debug!("{:?} // Dropped {:?} because the sender was not the leader {:?} vs {:?} (ours)",
+                        self.node_id,message, header.from(), view.leader());
 
                         return Ok(DecisionStatus::Deciding);
                     }
                     ConsensusMessageKind::PrePrepare(_)
                     if message.sequence_number() != self.seq => {
                         //Drop proposed value since it is not for this consensus instance
-                        warn!("{:?} // Dropped pre prepare message because the sequence number was not the same {:?} vs {:?} (ours)",
-                            self.node_id, message.sequence_number(), self.seq);
+                        warn!("{:?} // Dropped {:?} because the sequence number was not the same {:?} vs {:?} (ours)",
+                            self.node_id, message, message.sequence_number(), self.seq);
 
                         return Ok(DecisionStatus::Deciding);
                     }
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index c0e5562b..65a8d04b 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -190,6 +190,9 @@ impl<D, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL,
 
     fn poll(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request, QuorumJoinCert<RP>>, D::Request>
         where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+
+        trace!("{:?} // Polling {:?}", self.node.id(), self.phase);
+
         match self.phase {
             ConsensusPhase::NormalPhase => {
                 self.poll_normal_phase()
@@ -820,6 +823,8 @@ impl<D, NT, PL, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, NT,
           NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
           PL: Clone {
     fn attempt_network_view_change(&mut self, join_certificate: QuorumJoinCert<RP>) -> Result<ReconfigurationAttemptResult> {
+        self.switch_phase(ConsensusPhase::SyncPhase);
+
         Ok(self.synchronizer.attempt_join_quorum(join_certificate, &*self.node, &self.timeouts))
     }
 }
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 7fef9f4e..6c48ad77 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -6,7 +6,6 @@ use std::{
     time::Duration,
 };
 use std::cell::Cell;
-use std::collections::{BTreeMap, BTreeSet};
 use std::fmt::{Debug, Formatter};
 
 use either::Either;
@@ -224,7 +223,7 @@ impl<O, JC> TboQueue<O, JC> {
                 false
             }
             Either::Left(_) => {
-                unreachable!("How can we possibly go back in time?");
+                unreachable!("How can we possibly go back in time? View {:?} vs our current {:?}", view, self.view);
             }
         };
     }
@@ -258,15 +257,12 @@ impl<O, JC> TboQueue<O, JC> {
     /// Verifies if we have new `STOP` messages to be processed for
     /// the current view.
     pub fn can_process_stops(&self) -> bool {
-        self.stop
-            .get(0)
-            .map(|deque| deque.len() > 0)
-            .unwrap_or(false)
+        self.stop.get(0).map(|deque| deque.len() > 0).unwrap_or(false) ||
+            self.quorum_join.get(0).map(|deque| deque.len() > 0).unwrap_or(false)
     }
 
     fn queue_node_quorum_join(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
         let seq = self.view.sequence_number().next();
-
         tbo_queue_message(seq, &mut self.quorum_join, StoredMessage::new(h, m))
     }
 
@@ -428,8 +424,10 @@ impl<D, RP> AbstractSynchronizer<D, RP> for Synchronizer<D, RP>
         // FIXME: is the following line necessary?
         let mut guard = self.tbo.lock().unwrap();
 
-        if guard.install_view(view) {
+        if !guard.install_view(view) {
             // If we don't install a new view, then we don't want to forget our current state now do we?
+
+            debug!("Replacing our phase with Init");
             self.phase.replace(ProtoPhase::Init);
         }
     }
@@ -516,6 +514,9 @@ impl<D, RP> Synchronizer<D, RP>
                     &mut tbo_guard.stop
                 );
 
+                // Reset the tbo guard so we also query the join cert queue
+                tbo_guard.get_queue = true;
+
                 match result {
                     SynchronizerPollStatus::Recv => {
                         // if we don't have any pending stop messages, check the join quorum
@@ -528,10 +529,24 @@ impl<D, RP> Synchronizer<D, RP>
                 }
             }
             ProtoPhase::Stopping(_) | ProtoPhase::Stopping2(_) | ProtoPhase::ViewStopping(_) | ProtoPhase::ViewStopping2(_) => {
-                extract_msg!(D::Request, QuorumJoinCert<RP> =>
+                let result = extract_msg!(D::Request, QuorumJoinCert<RP> =>
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop
-                )
+                );
+
+                // Reset the tbo guard so we also query the join cert queue
+                tbo_guard.get_queue = true;
+
+                match result {
+                    SynchronizerPollStatus::Recv => {
+                        // if we don't have any pending stop messages, check the join quorum
+                        extract_msg!(D::Request, QuorumJoinCert<RP> =>
+                            &mut tbo_guard.get_queue, &mut tbo_guard.quorum_join)
+                    }
+                    _ => {
+                        result
+                    }
+                }
             }
             ProtoPhase::StoppingData(_) => {
                 extract_msg!(D::Request, QuorumJoinCert<RP>  =>
@@ -754,6 +769,23 @@ impl<D, RP> Synchronizer<D, RP>
                 let next_seq = current_view.sequence_number().next();
 
                 let (received, node_id, jc) = match message.kind() {
+                    ViewChangeMessageKind::NodeQuorumJoin(node_id, join_certificate) if msg_seq == next_seq => {
+                        //TODO: Verify
+
+                        self.currently_adding_node.replace(Some(*node_id));
+
+                        let message = ViewChangeMessageKind::StopQuorumJoin(*node_id, join_certificate.clone());
+
+                        let message = ViewChangeMessage::new(next_seq, message);
+
+                        let message = PBFTMessage::ViewChange(message);
+
+                        node.broadcast_signed(message, current_view.quorum_members().clone().into_iter());
+
+                        self.begin_quorum_view_change(Some(join_certificate.clone()), &**node, timeouts, log);
+
+                        return SynchronizerStatus::Running;
+                    }
                     ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
                         let mut guard = self.tbo.lock().unwrap();
 
@@ -783,8 +815,7 @@ impl<D, RP> Synchronizer<D, RP>
 
                         return SynchronizerStatus::Running;
                     }
-                    ViewChangeMessageKind::StopQuorumJoin(node, join_cert)
-                    if self.currently_adding_node.get().is_some() && self.currently_adding_node.get().unwrap() != *node => {
+                    ViewChangeMessageKind::StopQuorumJoin(node, join_cert) if self.currently_adding_node.get().is_some() && self.currently_adding_node.get().unwrap() != *node => {
                         error!("{:?} // Received stop quorum join message with node that does not match the current received ", node.id());
 
                         return SynchronizerStatus::Running;
@@ -826,6 +857,8 @@ impl<D, RP> Synchronizer<D, RP>
 
                 if let ProtoPhase::ViewStopping(_received) = self.phase.get() {
                     return if received > current_view.params().f() {
+                        self.currently_adding_node.replace(Some(node_id));
+
                         self.begin_quorum_view_change(None, &**node, timeouts, log);
 
                         SynchronizerStatus::Running
@@ -1270,6 +1303,8 @@ impl<D, RP> Synchronizer<D, RP>
         where NT: OrderProtocolSendNode<D, PBFT<D, RP>>,
               PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
     {
+        debug!("Beginning quorum view change with certificate {} at phase {:?}",  join_cert.is_some(), self.phase.get());
+
         match (self.phase.get(), &join_cert) {
             (ProtoPhase::ViewStopping(i), None) => {
                 self.phase.replace(ProtoPhase::ViewStopping2(i + 1));
@@ -1296,6 +1331,8 @@ impl<D, RP> Synchronizer<D, RP>
         where NT: OrderProtocolSendNode<D, PBFT<D, RP>>, {
         let current_view = self.view();
 
+        info!("{:?} // Attempting to join quorum", node.id());
+
         if current_view.quorum_members().contains(&node.id()) {
             //We are already a part of the quorum, so we don't need to do anything
             info!("{:?} // Attempted to join quorum, but we are already a part of it", node.id());

From 100d419338405c5a52f1e77c87a2974a8787cf42 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Tue, 1 Aug 2023 14:40:32 +0100
Subject: [PATCH 28/80] Fixed bug where reconfiguration protocol was left
 hanging since no response was delivered by the ordering protocol.

---
 febft-pbft-consensus/src/bft/mod.rs      | 14 +++++++--
 febft-pbft-consensus/src/bft/sync/mod.rs | 36 ++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 65a8d04b..149e6f07 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -77,6 +77,7 @@ pub enum SyncPhaseRes<O> {
     SyncProtocolNotNeeded,
     RunSyncProtocol,
     SyncProtocolFinished(Option<ProtocolConsensusDecision<O>>),
+    JoinedQuorum(Option<ProtocolConsensusDecision<O>>),
     RunCSTProtocol,
 }
 
@@ -190,7 +191,6 @@ impl<D, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL,
 
     fn poll(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request, QuorumJoinCert<RP>>, D::Request>
         where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
-
         trace!("{:?} // Polling {:?}", self.node.id(), self.phase);
 
         match self.phase {
@@ -396,7 +396,7 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
                             // As that has already been done by the adv sync method
                             return OrderProtocolPoll::RunCst;
                         }
-                        _ => {}
+                        _ => unreachable!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message")
                     }
                 } else {
                     // The synchronizer should never return anything other than a view
@@ -446,6 +446,9 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
                             }
                         }
                     }
+                    SyncPhaseRes::JoinedQuorum(to_execute) => {
+                        OrderProtocolExecResult::QuorumJoinResult(true, to_execute.map(|x| vec![x]))
+                    }
                     SyncPhaseRes::RunCSTProtocol => {
                         OrderProtocolExecResult::RunCst
                     }
@@ -608,6 +611,13 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
 
                 SyncPhaseRes::RunCSTProtocol
             }
+            SynchronizerStatus::NewViewJoinedQuorum(decision) => {
+                //We have joined a quorum and we have a new view to execute
+                //We need to switch to the normal phase and execute the new view
+                self.switch_phase(ConsensusPhase::NormalPhase);
+
+                SyncPhaseRes::JoinedQuorum(decision)
+            }
             // should not happen...
             _ => {
                 unreachable!()
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 6c48ad77..efd6a95d 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -339,6 +339,9 @@ pub enum SynchronizerStatus<O> {
     Running,
     /// The view change protocol just finished running.
     NewView(Option<ProtocolConsensusDecision<O>>),
+    /// The view change protocol just finished running and we
+    /// have successfully joined the quorum.
+    NewViewJoinedQuorum(Option<ProtocolConsensusDecision<O>>),
     /// Before we finish the view change protocol, we need
     /// to run the CST protocol.
     RunCst,
@@ -398,6 +401,9 @@ pub struct Synchronizer<D: ApplicationData, RP: ReconfigurationProtocolMessage>
     collects: Mutex<CollectsType<D, QuorumJoinCert<RP>>>,
     // Used to store the finalize state when we are forced to run the CST protocol
     finalize_state: RefCell<Option<FinalizeState<D::Request>>>,
+    // We need to keep track of whether we are entering the quorum
+    entering_quorum: Cell<bool>,
+    // Replica accessory
     accessory: SynchronizerAccessory<D>,
 }
 
@@ -450,6 +456,7 @@ impl<D, RP> Synchronizer<D, RP>
             collects: Mutex::new(Default::default()),
             tbo: Mutex::new(TboQueue::new(view)),
             finalize_state: RefCell::new(None),
+            entering_quorum: Cell::new(false),
             accessory: SynchronizerAccessory::Follower(FollowerSynchronizer::new()),
         })
     }
@@ -462,6 +469,7 @@ impl<D, RP> Synchronizer<D, RP>
             collects: Mutex::new(Default::default()),
             tbo: Mutex::new(TboQueue::new(view)),
             finalize_state: RefCell::new(None),
+            entering_quorum: Cell::new(false),
             accessory: SynchronizerAccessory::Replica(ReplicaSynchronizer::new(timeout_dur)),
         })
     }
@@ -481,6 +489,7 @@ impl<D, RP> Synchronizer<D, RP>
             currently_adding_node: Cell::new(None),
             collects: Mutex::new(Default::default()),
             finalize_state: RefCell::new(None),
+            entering_quorum: Cell::new(false),
             accessory: SynchronizerAccessory::Replica(ReplicaSynchronizer::new(timeout_dur)),
         }))
     }
@@ -509,11 +518,24 @@ impl<D, RP> Synchronizer<D, RP>
             ProtoPhase::Init => {
                 //If we are in the init phase and there is a pending request, move to the stopping phase
                 let result = extract_msg!(D::Request, QuorumJoinCert<RP>  =>
-                    { self.phase.replace(ProtoPhase::Stopping(0)); },
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop
                 );
 
+                match &result {
+                    SynchronizerPollStatus::NextMessage(h, vc) => {
+                        match vc.kind() {
+                            ViewChangeMessageKind::StopQuorumJoin(_, _) => {
+                                self.phase.replace(ProtoPhase::ViewStopping(0));
+                            }
+                            _ => {
+                                self.phase.replace(ProtoPhase::Stopping(0));
+                            }
+                        }
+                    }
+                    _ => {}
+                };
+
                 // Reset the tbo guard so we also query the join cert queue
                 tbo_guard.get_queue = true;
 
@@ -1357,6 +1379,8 @@ impl<D, RP> Synchronizer<D, RP>
             self.phase.replace(ProtoPhase::Syncing);
         }
 
+        self.entering_quorum.replace(true);
+
         self.install_view(view);
 
         return ReconfigurationAttemptResult::InProgress;
@@ -1418,6 +1442,7 @@ impl<D, RP> Synchronizer<D, RP>
                 self.stopped.borrow_mut().clear();
                 self.collects.lock().unwrap().clear();
                 self.currently_adding_node.replace(None);
+                self.entering_quorum.replace(false);
 
                 //Set the new state to be stopping
                 self.phase.replace(ProtoPhase::Stopping2(0));
@@ -1527,8 +1552,13 @@ impl<D, RP> Synchronizer<D, RP>
         self.tbo.lock().unwrap().next_instance_queue();
         self.phase.replace(ProtoPhase::Init);
 
-        // resume normal phase
-        SynchronizerStatus::NewView(to_execute)
+        if self.entering_quorum.get() && view.quorum_members().contains(&node.id()) {
+            self.entering_quorum.replace(false);
+            SynchronizerStatus::NewViewJoinedQuorum(to_execute)
+        } else {
+            // resume normal phase
+            SynchronizerStatus::NewView(to_execute)
+        }
     }
 
     /// Handle a batch of requests received from a Pre prepare message sent by the leader

From 60985bcd5699c55bf27da4c4de6332e6b9d7a8cc Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Tue, 1 Aug 2023 16:52:43 +0100
Subject: [PATCH 29/80] Fixed bugs with initialization of the quorum view
 change protocol, specially in where the stop quorum join message is bcast.

---
 febft-pbft-consensus/src/bft/sync/mod.rs      | 37 +++++++--------
 .../src/bft/sync/replica_sync/mod.rs          | 45 ++++++++++++++++++-
 2 files changed, 60 insertions(+), 22 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index efd6a95d..d30d772c 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -395,7 +395,7 @@ pub struct Synchronizer<D: ApplicationData, RP: ReconfigurationProtocolMessage>
     //Stores currently received requests from other nodes
     stopped: RefCell<IntMap<Vec<StoredRequestMessage<D::Request>>>>,
     //Stores currently received requests from other nodes
-    currently_adding_node: Cell<Option<NodeId>>,
+    currently_adding_node: Cell<Option<(NodeId, QuorumJoinCert<RP>)>>,
     //TODO: This does not require a Mutex I believe since it's only accessed when
     // Processing messages (which is always done in the replica thread)
     collects: Mutex<CollectsType<D, QuorumJoinCert<RP>>>,
@@ -732,9 +732,7 @@ impl<D, RP> Synchronizer<D, RP>
 
                 // FIXME: Check if we have already seen the messages in the stop quorum
 
-                self.stopped
-                    .borrow_mut()
-                    .insert(header.from().into(), stopped);
+                self.stopped.borrow_mut().insert(header.from().into(), stopped);
 
                 // NOTE: we only take this branch of the code before
                 // we have sent our own STOP message
@@ -792,19 +790,11 @@ impl<D, RP> Synchronizer<D, RP>
 
                 let (received, node_id, jc) = match message.kind() {
                     ViewChangeMessageKind::NodeQuorumJoin(node_id, join_certificate) if msg_seq == next_seq => {
-                        //TODO: Verify
+                        //TODO: Verify node join certificate
 
-                        self.currently_adding_node.replace(Some(*node_id));
+                        self.currently_adding_node.replace(Some((*node_id, join_certificate.clone())));
 
-                        let message = ViewChangeMessageKind::StopQuorumJoin(*node_id, join_certificate.clone());
-
-                        let message = ViewChangeMessage::new(next_seq, message);
-
-                        let message = PBFTMessage::ViewChange(message);
-
-                        node.broadcast_signed(message, current_view.quorum_members().clone().into_iter());
-
-                        self.begin_quorum_view_change(Some(join_certificate.clone()), &**node, timeouts, log);
+                        self.begin_quorum_view_change(Some((*node_id, join_certificate.clone())), &**node, timeouts, log);
 
                         return SynchronizerStatus::Running;
                     }
@@ -879,7 +869,7 @@ impl<D, RP> Synchronizer<D, RP>
 
                 if let ProtoPhase::ViewStopping(_received) = self.phase.get() {
                     return if received > current_view.params().f() {
-                        self.currently_adding_node.replace(Some(node_id));
+                        self.currently_adding_node.replace(Some((node_id, jc.clone())));
 
                         self.begin_quorum_view_change(None, &**node, timeouts, log);
 
@@ -892,7 +882,7 @@ impl<D, RP> Synchronizer<D, RP>
                 }
 
                 if received >= current_view.params().quorum() {
-                    let next_view = current_view.next_view_with_new_node(self.currently_adding_node.get().unwrap());
+                    let next_view = current_view.next_view_with_new_node(self.currently_adding_node.get().unwrap().0);
 
                     let previous_view = current_view.clone();
 
@@ -1318,7 +1308,7 @@ impl<D, RP> Synchronizer<D, RP>
 
     pub fn begin_quorum_view_change<NT, PL>(
         &self,
-        join_cert: Option<QuorumJoinCert<RP>>,
+        join_cert: Option<(NodeId, QuorumJoinCert<RP>)>,
         node: &NT,
         timeouts: &Timeouts,
         _log: &Log<D, PL>, )
@@ -1337,7 +1327,7 @@ impl<D, RP> Synchronizer<D, RP>
                 // The amount of messages received (since we have not actually received any messages)
                 self.phase.replace(ProtoPhase::ViewStopping2(i));
             }
-            (ProtoPhase::StoppingData(_), _) | (ProtoPhase::Syncing, _) | (ProtoPhase::Stopping2(_), _) => {
+            (ProtoPhase::StoppingData(_), _) | (ProtoPhase::Syncing, _) | (ProtoPhase::ViewStopping2(_), _) => {
                 // we have already started a view change protocol
                 return;
             }
@@ -1345,6 +1335,13 @@ impl<D, RP> Synchronizer<D, RP>
                 self.currently_adding_node.replace(None);
             }
         }
+
+        match &self.accessory {
+            SynchronizerAccessory::Follower(_) => {}
+            SynchronizerAccessory::Replica(replica) => {
+                replica.handle_begin_quorum_view_change(self, timeouts, node, join_cert)
+            }
+        }
     }
 
     pub fn attempt_join_quorum<NT>(&self, join_certificate: QuorumJoinCert<RP>,
@@ -1429,7 +1426,7 @@ impl<D, RP> Synchronizer<D, RP>
                 self.phase.replace(ProtoPhase::Stopping2(0));
             }
             (ProtoPhase::StoppingData(_), _) | (ProtoPhase::Syncing, _) | (ProtoPhase::Stopping2(_), _) => {
-                // we have already started a view change protocol
+                // we have already started a view change protocol or we have already sent our STOP message
                 return;
             }
             // we have timed out, therefore we should send a STOP msg;
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index fdd8a6c5..edd4bb52 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -9,6 +9,7 @@ use std::marker::PhantomData;
 
 use std::time::{Duration, Instant};
 use log::{debug, error, info};
+use num_traits::real::Real;
 use atlas_common::collections;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable};
@@ -18,6 +19,7 @@ use atlas_execution::serialize::ApplicationData;
 use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, RequestMessage, StoredRequestMessage, SystemMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
+use atlas_core::reconfiguration_protocol::QuorumJoinCert;
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
 use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, TimeoutPhase, Timeouts};
@@ -123,8 +125,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
 
         // broadcast STOP message with pending requests collected
         // from peer nodes' STOP messages
-        let requests = self.stopped_requests(base_sync,
-                                             timed_out);
+        let requests = self.stopped_requests(base_sync, timed_out);
 
         let current_view = base_sync.view();
 
@@ -143,6 +144,30 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         node.broadcast(message, targets.into_iter());
     }
 
+    pub(super) fn handle_begin_quorum_view_change<NT, RP>(
+        &self,
+        base_sync: &Synchronizer<D, RP>,
+        timeouts: &Timeouts,
+        node: &NT,
+        join_cert: Option<(NodeId, QuorumJoinCert<RP>)>,
+    ) where RP: ReconfigurationProtocolMessage + 'static,
+            NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
+
+        let current_view = base_sync.view();
+
+        let (node_id, join_certificate) = self.joining_node(base_sync, join_cert);
+
+        info!("{:?} // Beginning a quorum view change to next view with new node: {:?}", node.id(), node_id);
+
+        let message = ViewChangeMessageKind::StopQuorumJoin(node_id, join_certificate);
+
+        let message = ViewChangeMessage::new(current_view.sequence_number().next(), message);
+
+        let message = PBFTMessage::ViewChange(message);
+
+        node.broadcast_signed(message, current_view.quorum_members().clone().into_iter());
+    }
+
     /// Watch a vector of requests received
     pub fn watch_received_requests(
         &self,
@@ -381,6 +406,22 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         all_reqs.drain().map(|(_, stop)| stop).collect()
     }
 
+    fn joining_node<RP>(
+        &self,
+        base_sync: &Synchronizer<D, RP>,
+        join_cert: Option<(NodeId, QuorumJoinCert<RP>)>,
+    ) -> (NodeId, QuorumJoinCert<RP>)
+        where RP: ReconfigurationProtocolMessage + 'static {
+        let (node, join_cert) = match join_cert {
+            Some((node, join_cert)) => (node, join_cert),
+            None => {
+                let adding_node = base_sync.currently_adding_node.get();
+
+                adding_node.expect("We should have a node we are adding")
+            }
+        };
+    }
+
     fn stopped_request_digests<RP>(
         &self,
         base_sync: &Synchronizer<D, RP>,

From 432574b9a9a7bff1ac41c9dbd1d183db84524587 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 2 Aug 2023 16:09:07 +0100
Subject: [PATCH 30/80] Reworked how the quorum adds a node and removed its
 dependence on the reconfiguration protocol. Now it's the RPs job to check the
 validity of join certificates and then instruct the ordering protocol to
 start the quorum join procedure.

---
 febft-pbft-consensus/src/bft/config/mod.rs    |  11 +-
 .../src/bft/consensus/accessory/mod.rs        |  39 +-
 .../bft/consensus/accessory/replica/mod.rs    |  38 +-
 .../src/bft/consensus/decision/mod.rs         |  26 +-
 febft-pbft-consensus/src/bft/consensus/mod.rs |  51 +-
 febft-pbft-consensus/src/bft/message/mod.rs   |  71 ++-
 .../src/bft/message/serialize/mod.rs          |  13 +-
 febft-pbft-consensus/src/bft/mod.rs           | 153 +++---
 .../src/bft/msg_log/decided_log/mod.rs        |  34 +-
 febft-pbft-consensus/src/bft/proposer/mod.rs  |  23 +-
 febft-pbft-consensus/src/bft/sync/mod.rs      | 502 ++++++++----------
 .../src/bft/sync/replica_sync/mod.rs          | 100 ++--
 12 files changed, 470 insertions(+), 591 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/config/mod.rs b/febft-pbft-consensus/src/bft/config/mod.rs
index 5dc35247..f6f4c007 100644
--- a/febft-pbft-consensus/src/bft/config/mod.rs
+++ b/febft-pbft-consensus/src/bft/config/mod.rs
@@ -9,21 +9,20 @@ use atlas_execution::serialize::ApplicationData;
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::sync::view::ViewInfo;
 
-pub struct PBFTConfig<D, RP>
-    where D: ApplicationData, RP: ReconfigurationProtocolMessage + 'static {
+pub struct PBFTConfig<D>
+    where D: ApplicationData{
     pub node_id: NodeId,
     // pub observer_handle: ObserverHandle,
-    pub follower_handle: Option<FollowerHandle<PBFTConsensus<D, RP>>>,
+    pub follower_handle: Option<FollowerHandle<PBFTConsensus<D>>>,
     pub view: ViewInfo,
     pub timeout_dur: Duration,
     pub proposer_config: ProposerConfig,
     pub watermark: u32,
 }
 
-impl<D: ApplicationData + 'static,
-    RP: ReconfigurationProtocolMessage + 'static> PBFTConfig<D, RP> {
+impl<D: ApplicationData + 'static,> PBFTConfig<D> {
     pub fn new(node_id: NodeId,
-               follower_handle: Option<FollowerHandle<PBFTConsensus<D, RP>>>,
+               follower_handle: Option<FollowerHandle<PBFTConsensus<D>>>,
                view: ViewInfo, timeout_dur: Duration,
                watermark: u32, proposer_config: ProposerConfig) -> Self {
         Self {
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
index 507c08c7..dc663849 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
@@ -13,62 +13,59 @@ use crate::bft::sync::view::ViewInfo;
 
 pub mod replica;
 
-pub enum ConsensusDecisionAccessory<D, RP>
-    where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static {
+pub enum ConsensusDecisionAccessory<D>
+    where D: ApplicationData + 'static, {
     Follower,
-    Replica(ReplicaAccessory<D, RP>),
+    Replica(ReplicaAccessory<D>),
 }
 
-pub trait AccessoryConsensus<D,RP> where D: ApplicationData + 'static,
-                                                  RP: ReconfigurationProtocolMessage + 'static {
+pub trait AccessoryConsensus<D> where D: ApplicationData + 'static,{
     /// Handle the reception of a pre-prepare message without having completed the pre prepare phase
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D,RP>> + 'static;
+                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle the prepare phase having been completed
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D,RP>> + 'static;
+                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a prepare message processed during the preparing phase without having
     /// reached a quorum
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static;
+                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a prepare message processed during the prepare phase when a quorum
     /// has been achieved
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
-                                   node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static;
+                                   node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a commit message processed during the preparing phase without having
     /// reached a quorum
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
-                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static;
+                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a commit message processed during the prepare phase when a quorum has been reached
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
-                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static;
+                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 }
 
-impl<D,  RP> AccessoryConsensus<D,  RP> for ConsensusDecisionAccessory<D, RP>
-    where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static {
+impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
+    where D: ApplicationData + 'static{
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static {
+                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -80,7 +77,7 @@ impl<D,  RP> AccessoryConsensus<D,  RP> for ConsensusDecisionAccessory<D, RP>
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static{
+                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static{
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -92,7 +89,7 @@ impl<D,  RP> AccessoryConsensus<D,  RP> for ConsensusDecisionAccessory<D, RP>
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static {
+                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -104,7 +101,7 @@ impl<D,  RP> AccessoryConsensus<D,  RP> for ConsensusDecisionAccessory<D, RP>
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
-                                   node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static {
+                                   node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -116,7 +113,7 @@ impl<D,  RP> AccessoryConsensus<D,  RP> for ConsensusDecisionAccessory<D, RP>
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
-                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static {
+                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
@@ -128,7 +125,7 @@ impl<D,  RP> AccessoryConsensus<D,  RP> for ConsensusDecisionAccessory<D, RP>
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
-                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D, RP>> + 'static {
+                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index 457ae4a8..b998f1fa 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -24,25 +24,23 @@ use crate::bft::{PBFT, SysMsg};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::sync::view::ViewInfo;
 
-pub struct ReplicaAccessory<D, RP>
-    where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static {
-    speculative_commits: Arc<Mutex<BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, RP>>>>>,
+pub struct ReplicaAccessory<D>
+    where D: ApplicationData + 'static {
+    speculative_commits: Arc<Mutex<BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D>>>>>,
 }
 
-impl<D, RP> AccessoryConsensus<D, RP> for ReplicaAccessory<D, RP>
-    where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static {
+impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
+    where D: ApplicationData + 'static,{
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
-                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {}
+                                      node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 
     fn handle_pre_prepare_phase_completed<NT>(&mut self,
                                               deciding_log: &DecidingLog<D::Request>,
                                               view: &ViewInfo,
                                               _msg: StoredConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static {
+                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
         let my_id = node.id();
         let view_seq = view.sequence_number();
         let current_digest = deciding_log.current_digest().unwrap();
@@ -117,18 +115,18 @@ impl<D, RP> AccessoryConsensus<D, RP> for ReplicaAccessory<D, RP>
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {}
+                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                    view: &ViewInfo,
-                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
+                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {
         let node_id = node.id();
 
         let seq = deciding_log.sequence_number();
         let current_digest = deciding_log.current_digest().unwrap();
         let speculative_commits = self.take_speculative_commits();
 
-        if valid_spec_commits::<D, RP>(&speculative_commits, node_id, seq, view) {
+        if valid_spec_commits::<D>(&speculative_commits, node_id, seq, view) {
             for (_, msg) in speculative_commits.iter() {
                 debug!("{:?} // Broadcasting speculative commit message (total of {} messages) to {} targets",
                      node_id, speculative_commits.len(), view.params().n());
@@ -159,23 +157,22 @@ impl<D, RP> AccessoryConsensus<D, RP> for ReplicaAccessory<D, RP>
 
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                        view: &ViewInfo,
-                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D,  RP>> {}
+                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
                                     view: &ViewInfo,
-                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D,  RP>> {}
+                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 }
 
-impl<D,  RP> ReplicaAccessory<D, RP>
-    where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static {
+impl<D> ReplicaAccessory<D>
+    where D: ApplicationData + 'static, {
     pub fn new() -> Self {
         Self {
             speculative_commits: Arc::new(Mutex::new(BTreeMap::new())),
         }
     }
 
-    fn take_speculative_commits(&self) -> BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D,  RP>>> {
+    fn take_speculative_commits(&self) -> BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D>>> {
         let mut map = self.speculative_commits.lock().unwrap();
         std::mem::replace(&mut *map, BTreeMap::new())
     }
@@ -183,15 +180,14 @@ impl<D,  RP> ReplicaAccessory<D, RP>
 
 
 #[inline]
-fn valid_spec_commits<D,  RP>(
-    speculative_commits: &BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D, RP>>>,
+fn valid_spec_commits<D>(
+    speculative_commits: &BTreeMap<NodeId, StoredSerializedProtocolMessage<SysMsg<D>>>,
     node_id: NodeId,
     seq_no: SeqNo,
     view: &ViewInfo,
 ) -> bool
     where
         D: ApplicationData + 'static,
-        RP: ReconfigurationProtocolMessage + 'static
 {
     let len = speculative_commits.len();
 
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 887dfee3..957f4ed4 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -105,8 +105,7 @@ pub struct MessageQueue<O> {
 }
 
 /// The information needed to make a decision on a batch of requests.
-pub struct ConsensusDecision<D, PL, RP> where D: ApplicationData + 'static,
-                                              RP: ReconfigurationProtocolMessage + 'static {
+pub struct ConsensusDecision<D, PL> where D: ApplicationData + 'static, {
     node_id: NodeId,
     /// The sequence number of this consensus decision
     seq: SeqNo,
@@ -119,7 +118,7 @@ pub struct ConsensusDecision<D, PL, RP> where D: ApplicationData + 'static,
     /// Persistent log reference
     persistent_log: PL,
     /// Accessory to the base consensus state machine
-    accessory: ConsensusDecisionAccessory<D, RP>,
+    accessory: ConsensusDecisionAccessory<D>,
     // Metrics about the consensus instance
     consensus_metrics: ConsensusMetrics,
     //TODO: Store things directly into the persistent log as well as delete them when
@@ -175,9 +174,8 @@ impl<O> MessageQueue<O> {
     }
 }
 
-impl<D, PL, RP> ConsensusDecision<D, PL, RP>
-    where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static {
+impl<D, PL> ConsensusDecision<D, PL>
+    where D: ApplicationData + 'static,{
     pub fn init_decision(node_id: NodeId, seq_no: SeqNo, view: &ViewInfo, persistent_log: PL) -> Self {
         Self {
             node_id,
@@ -265,12 +263,12 @@ impl<D, PL, RP> ConsensusDecision<D, PL, RP>
     pub fn process_message<NT>(&mut self,
                                header: Header,
                                message: ConsensusMessage<D::Request>,
-                               synchronizer: &Synchronizer<D, RP>,
+                               synchronizer: &Synchronizer<D>,
                                timeouts: &Timeouts,
                                log: &mut Log<D, PL>,
                                node: &Arc<NT>) -> Result<DecisionStatus>
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+        where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let view = synchronizer.view();
 
         return match self.phase {
@@ -594,24 +592,22 @@ impl<D, PL, RP> ConsensusDecision<D, PL, RP>
     }
 }
 
-impl<D, PL, RP> Orderable for ConsensusDecision<D, PL, RP>
-    where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static {
+impl<D, PL> Orderable for ConsensusDecision<D, PL>
+    where D: ApplicationData + 'static, {
     fn sequence_number(&self) -> SeqNo {
         self.seq
     }
 }
 
 #[inline]
-fn request_batch_received<D, RP>(
+fn request_batch_received<D>(
     pre_prepare: &StoredConsensusMessage<D::Request>,
     timeouts: &Timeouts,
-    synchronizer: &Synchronizer<D, RP>,
+    synchronizer: &Synchronizer<D>,
     log: &DecidingLog<D::Request>,
 ) -> Vec<ClientRqInfo>
     where
         D: ApplicationData + 'static,
-        RP: ReconfigurationProtocolMessage + 'static
 {
     let start = Instant::now();
 
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 5a0518d6..4ded90b8 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -185,9 +185,8 @@ pub struct Signals {
 
 /// The consensus handler. Responsible for multiplexing consensus instances and keeping track
 /// of missing messages
-pub struct Consensus<D, PL, RP>
+pub struct Consensus<D, PL>
     where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static,
           PL: Clone {
     node_id: NodeId,
     /// The handle to the executor of the function
@@ -203,7 +202,7 @@ pub struct Consensus<D, PL, RP>
     /// The consensus instances that are currently being processed
     /// A given consensus instance n will only be finished when all consensus instances
     /// j, where j < n have already been processed, in order to maintain total ordering
-    decisions: VecDeque<ConsensusDecision<D, PL, RP>>,
+    decisions: VecDeque<ConsensusDecision<D, PL>>,
     /// The queue for messages that sit outside the range seq_no + watermark
     /// These messages cannot currently be processed since they sit outside the allowed
     /// zone but they will be processed once the seq no moves forward enough to include them
@@ -222,9 +221,8 @@ pub struct Consensus<D, PL, RP>
     persistent_log: PL,
 }
 
-impl<D, PL, RP> Consensus<D, PL, RP> where D: ApplicationData + 'static,
-                                           RP: ReconfigurationProtocolMessage + 'static,
-                                           PL: Clone {
+impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
+                                   PL: Clone {
     pub fn new_replica(node_id: NodeId, view: &ViewInfo, executor_handle: ExecutorHandle<D>, seq_no: SeqNo,
                        watermark: u32, consensus_guard: Arc<ProposerConsensusGuard>, timeouts: Timeouts,
                        persistent_log: PL) -> Self {
@@ -365,12 +363,12 @@ impl<D, PL, RP> Consensus<D, PL, RP> where D: ApplicationData + 'static,
     pub fn process_message<NT>(&mut self,
                                header: Header,
                                message: ConsensusMessage<D::Request>,
-                               synchronizer: &Synchronizer<D, RP>,
+                               synchronizer: &Synchronizer<D>,
                                timeouts: &Timeouts,
                                log: &mut Log<D, PL>,
                                node: &Arc<NT>) -> Result<ConsensusStatus>
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>, {
+        where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D>>, {
         let message_seq = message.sequence_number();
 
         let view_seq = message.view();
@@ -482,7 +480,7 @@ impl<D, PL, RP> Consensus<D, PL, RP> where D: ApplicationData + 'static,
     /// Advance to the next instance of the consensus
     /// This will also create the necessary new decision to keep the pending decisions
     /// equal to the water mark
-    pub fn next_instance(&mut self, view: &ViewInfo) -> ConsensusDecision<D, PL, RP> {
+    pub fn next_instance(&mut self, view: &ViewInfo) -> ConsensusDecision<D, PL> {
         let decision = self.decisions.pop_front().unwrap();
 
         self.seq_no = self.seq_no.next();
@@ -699,8 +697,7 @@ impl<D, PL, RP> Consensus<D, PL, RP> where D: ApplicationData + 'static,
                               view: &ViewInfo,
                               proof: Proof<D::Request>,
                               log: &mut Log<D, PL>) -> Result<ProtocolConsensusDecision<D::Request>>
-        where
-            PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
 
         // If this is successful, it means that we are all caught up and can now start executing the
         // batch
@@ -714,17 +711,15 @@ impl<D, PL, RP> Consensus<D, PL, RP> where D: ApplicationData + 'static,
 
     /// Create a fake `PRE-PREPARE`. This is useful during the view
     /// change protocol.
-    pub fn forge_propose<K>(
+    pub fn forge_propose(
         &self,
         requests: Vec<StoredRequestMessage<D::Request>>,
-        synchronizer: &K,
-    ) -> SysMsg<D, RP>
-        where
-            K: AbstractSynchronizer<D, RP>,
+        view: &ViewInfo,
+    ) -> SysMsg<D>
     {
         PBFTMessage::Consensus(ConsensusMessage::new(
             self.sequence_number(),
-            synchronizer.view().sequence_number(),
+            view.sequence_number(),
             ConsensusMessageKind::PrePrepare(requests),
         ))
     }
@@ -801,22 +796,17 @@ impl<D, PL, RP> Consensus<D, PL, RP> where D: ApplicationData + 'static,
     pub fn finalize_view_change<NT>(
         &mut self,
         (header, message): (Header, ConsensusMessage<D::Request>),
-        synchronizer: &Synchronizer<D, RP>,
+        new_view: &ViewInfo,
+        synchronizer: &Synchronizer<D>,
         timeouts: &Timeouts,
         log: &mut Log<D, PL>,
         node: &Arc<NT>,
     ) where
-        NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
-        PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
-        let view = synchronizer.view();
+        NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
+        PL: OrderingProtocolLog<PBFTConsensus<D>> {
         //Prepare the algorithm as we are already entering this phase
 
-        //TODO: when we finalize a view change, we want to treat the pre prepare request
-        // As the only pre prepare, since it already has info provided by everyone in the network.
-        // Therefore, this should go straight to the Preparing phase instead of waiting for
-        // All the view's leaders.
-
-        self.install_view(&view);
+        self.install_view(new_view);
 
         if let ConsensusMessageKind::PrePrepare(reqs) = &message.kind() {
             let mut final_rqs = Vec::with_capacity(reqs.len());
@@ -849,7 +839,7 @@ impl<D, PL, RP> Consensus<D, PL, RP> where D: ApplicationData + 'static,
     }
 
     /// Enqueue a decision onto our overlapping decision log
-    fn enqueue_decision(&mut self, decision: ConsensusDecision<D, PL, RP>) {
+    fn enqueue_decision(&mut self, decision: ConsensusDecision<D, PL>) {
         self.signalled.push_signalled(decision.sequence_number());
 
         self.decisions.push_back(decision);
@@ -871,9 +861,8 @@ impl<D, PL, RP> Consensus<D, PL, RP> where D: ApplicationData + 'static,
     }
 }
 
-impl<D, PL, RP> Orderable for Consensus<D, PL, RP>
+impl<D, PL> Orderable for Consensus<D, PL>
     where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static,
           PL: Clone {
     fn sequence_number(&self) -> SeqNo {
         self.seq_no
diff --git a/febft-pbft-consensus/src/bft/message/mod.rs b/febft-pbft-consensus/src/bft/message/mod.rs
index 5e99114e..86d6a6f7 100644
--- a/febft-pbft-consensus/src/bft/message/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/mod.rs
@@ -2,31 +2,25 @@
 //! between the system processes.
 
 use std::fmt::{Debug, Formatter};
-use std::io;
 use std::io::Write;
-use std::mem::MaybeUninit;
-use bytes::Bytes;
-
-#[cfg(feature = "serialize_serde")]
-use serde::{Serialize, Deserialize};
-
-use atlas_common::error::*;
 
 use futures::io::{
-    AsyncWriteExt,
     AsyncWrite,
+    AsyncWriteExt,
 };
-use atlas_common::crypto::hash::{Context, Digest};
-use atlas_common::crypto::signature::{KeyPair, PublicKey, Signature};
+#[cfg(feature = "serialize_serde")]
+use serde::{Deserialize, Serialize};
+
+use atlas_common::crypto::hash::Digest;
+use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_communication::message::{Header, NetworkMessage, NetworkMessageKind, PingMessage, StoredMessage};
-use atlas_execution::serialize::ApplicationData;
+use atlas_communication::message::Header;
 use atlas_core::messages::{RequestMessage, StoredRequestMessage};
+use atlas_execution::serialize::ApplicationData;
 
-use crate::bft::sync::LeaderCollects;
 use crate::bft::msg_log::decisions::CollectData;
-use crate::bft::PBFT;
+use crate::bft::sync::LeaderCollects;
 use crate::bft::sync::view::ViewInfo;
 
 pub mod serialize;
@@ -34,16 +28,16 @@ pub mod serialize;
 /// PBFT protocol messages
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub enum PBFTMessage<R, JC> {
+pub enum PBFTMessage<R> {
     /// Consensus message
     Consensus(ConsensusMessage<R>),
     /// View change messages
-    ViewChange(ViewChangeMessage<R, JC>),
+    ViewChange(ViewChangeMessage<R>),
     //Observer related messages
     ObserverMessage(ObserverMessage),
 }
 
-impl<R, JC> Debug for PBFTMessage<R, JC> {
+impl<R> Debug for PBFTMessage<R> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
             PBFTMessage::Consensus(_) => {
@@ -59,7 +53,7 @@ impl<R, JC> Debug for PBFTMessage<R, JC> {
     }
 }
 
-impl<R, JC> Orderable for PBFTMessage<R, JC> {
+impl<R> Orderable for PBFTMessage<R> {
     fn sequence_number(&self) -> SeqNo {
         match self {
             PBFTMessage::Consensus(consensus) => {
@@ -75,7 +69,7 @@ impl<R, JC> Orderable for PBFTMessage<R, JC> {
     }
 }
 
-impl<R, JC> PBFTMessage<R, JC> {
+impl<R> PBFTMessage<R> {
     pub fn consensus(&self) -> &ConsensusMessage<R> {
         match self {
             PBFTMessage::Consensus(msg) => msg,
@@ -90,14 +84,14 @@ impl<R, JC> PBFTMessage<R, JC> {
         }
     }
 
-    pub fn view_change(&self) -> &ViewChangeMessage<R, JC> {
+    pub fn view_change(&self) -> &ViewChangeMessage<R> {
         match self {
             PBFTMessage::ViewChange(msg) => msg,
             _ => panic!("Not a view change message"),
         }
     }
 
-    pub fn into_view_change(self) -> ViewChangeMessage<R, JC> {
+    pub fn into_view_change(self) -> ViewChangeMessage<R> {
         match self {
             PBFTMessage::ViewChange(msg) => msg,
             _ => panic!("Not a view change message"),
@@ -121,43 +115,43 @@ impl<R, JC> PBFTMessage<R, JC> {
 
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub struct ViewChangeMessage<O, JC> {
+pub struct ViewChangeMessage<O> {
     view: SeqNo,
-    kind: ViewChangeMessageKind<O, JC>,
+    kind: ViewChangeMessageKind<O>,
 }
 
-impl<O, JC> Orderable for ViewChangeMessage<O, JC> {
+impl<O> Orderable for ViewChangeMessage<O> {
     /// Returns the sequence number of the view this message refers to.
     fn sequence_number(&self) -> SeqNo {
         self.view
     }
 }
 
-impl<O, JC> Debug for ViewChangeMessage<O, JC> {
+impl<O> Debug for ViewChangeMessage<O> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(f, "View {:?}, {:?}", self.view, self.kind)
     }
 }
 
-impl<O, JC> ViewChangeMessage<O, JC> {
+impl<O> ViewChangeMessage<O> {
     /// Creates a new `ViewChangeMessage`, pertaining to the view
     /// with sequence number `view`, and of the kind `kind`.
-    pub fn new(view: SeqNo, kind: ViewChangeMessageKind<O, JC>) -> Self {
+    pub fn new(view: SeqNo, kind: ViewChangeMessageKind<O>) -> Self {
         Self { view, kind }
     }
 
     /// Returns a reference to the view change message kind.
-    pub fn kind(&self) -> &ViewChangeMessageKind<O, JC> {
+    pub fn kind(&self) -> &ViewChangeMessageKind<O> {
         &self.kind
     }
 
     /// Returns an owned view change message kind.
-    pub fn into_kind(self) -> ViewChangeMessageKind<O, JC> {
+    pub fn into_kind(self) -> ViewChangeMessageKind<O> {
         self.kind
     }
 
     /// Takes the collects embedded in this view change message, if they are available.
-    pub fn take_collects(self) -> Option<LeaderCollects<O, JC>> {
+    pub fn take_collects(self) -> Option<LeaderCollects<O>> {
         match self.kind {
             ViewChangeMessageKind::Sync(collects) => Some(collects),
             _ => {
@@ -169,16 +163,14 @@ impl<O, JC> ViewChangeMessage<O, JC> {
 
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub enum ViewChangeMessageKind<O, JC> {
-    /// A message, broadcast by the node attempting to join the quorum
-    NodeQuorumJoin(NodeId, JC),
+pub enum ViewChangeMessageKind<O> {
     /// A STOP message, broadcast when we want to call a view change due to requests getting timed out
     Stop(Vec<StoredRequestMessage<O>>),
     /// A STOP message, broadcast when we want to call a view change due to us having received a Node Quorum Join message
-    StopQuorumJoin(NodeId, JC),
+    StopQuorumJoin(NodeId),
     // Each of the latest decisions from the sender, so the new leader can sync
     StopData(CollectData<O>),
-    Sync(LeaderCollects<O, JC>),
+    Sync(LeaderCollects<O>),
 }
 
 /// Represents a message from the consensus sub-protocol.
@@ -432,12 +424,9 @@ impl Debug for ObserveEventKind {
     }
 }
 
-impl<O, JC> Debug for ViewChangeMessageKind<O, JC> {
+impl<O> Debug for ViewChangeMessageKind<O> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
-            ViewChangeMessageKind::NodeQuorumJoin(node, _) => {
-                write!(f, "Node quorum join {:?}", node)
-            }
             ViewChangeMessageKind::Stop(_) => {
                 write!(f, "Stop message")
             }
@@ -447,7 +436,7 @@ impl<O, JC> Debug for ViewChangeMessageKind<O, JC> {
             ViewChangeMessageKind::Sync(_) => {
                 write!(f, "Sync message")
             }
-            ViewChangeMessageKind::StopQuorumJoin(node, _) => {
+            ViewChangeMessageKind::StopQuorumJoin(node ) => {
                 write!(f, "Stop quorum join message {:?}", node)
             }
         }
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index e133ead0..0ecc4668 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -62,13 +62,12 @@ pub fn deserialize_consensus<R, D>(r: R) -> Result<ConsensusMessage<D::Request>>
 }
 
 /// The serializable type, to be used to appease the compiler and it's requirements
-pub struct PBFTConsensus<D: ApplicationData, RP: ReconfigurationProtocolMessage>(PhantomData<(D, RP)>);
+pub struct PBFTConsensus<D: ApplicationData>(PhantomData<(D)>);
 
-impl<D, RP> OrderingProtocolMessage for PBFTConsensus<D, RP>
-    where D: ApplicationData,
-          RP: ReconfigurationProtocolMessage {
+impl<D> OrderingProtocolMessage for PBFTConsensus<D>
+    where D: ApplicationData, {
     type ViewInfo = ViewInfo;
-    type ProtocolMessage = PBFTMessage<D::Request, QuorumJoinCert<RP>>;
+    type ProtocolMessage = PBFTMessage<D::Request>;
     type LoggableMessage = ConsensusMessage<D::Request>;
     type Proof = Proof<D::Request>;
     type ProofMetadata = ProofMetadata;
@@ -104,8 +103,8 @@ impl<D, RP> OrderingProtocolMessage for PBFTConsensus<D, RP>
     }
 }
 
-impl<D, RP> StatefulOrderProtocolMessage for PBFTConsensus<D, RP>
-    where D: ApplicationData, RP: ReconfigurationProtocolMessage {
+impl<D> StatefulOrderProtocolMessage for PBFTConsensus<D>
+    where D: ApplicationData + 'static {
     type DecLog = DecisionLog<D::Request>;
 
     #[cfg(feature = "serialize_capnp")]
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 149e6f07..2b880418 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -18,6 +18,7 @@ use log4rs::{
 
 use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
+use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
@@ -58,9 +59,9 @@ pub mod observer;
 pub mod metric;
 
 // The types responsible for this protocol
-pub type PBFT<D, RP> = PBFTConsensus<D, RP>;
+pub type PBFT<D> = PBFTConsensus<D>;
 // The message type for this consensus protocol
-pub type SysMsg<D, RP> = <PBFTConsensus<D, RP> as OrderingProtocolMessage>::ProtocolMessage;
+pub type SysMsg<D> = <PBFTConsensus<D> as OrderingProtocolMessage>::ProtocolMessage;
 
 #[derive(Clone, PartialEq, Eq, Debug)]
 /// Which phase of the consensus algorithm are we currently executing
@@ -82,18 +83,17 @@ pub enum SyncPhaseRes<O> {
 }
 
 /// a PBFT based ordering protocol
-pub struct PBFTOrderProtocol<D, NT, PL, RP>
+pub struct PBFTOrderProtocol<D, NT, PL>
     where
         D: ApplicationData + 'static,
-        RP: ReconfigurationProtocolMessage + 'static,
-        NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
+        NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
         PL: Clone {
     // What phase of the consensus algorithm are we currently executing
     phase: ConsensusPhase,
     /// The consensus state machine
-    consensus: Consensus<D, PL, RP>,
+    consensus: Consensus<D, PL>,
     /// The synchronizer state machine
-    synchronizer: Arc<Synchronizer<D, RP>>,
+    synchronizer: Arc<Synchronizer<D>>,
     /// The request pre processor
     pre_processor: RequestPreProcessor<D::Request>,
     // A reference to the timeouts layer
@@ -108,42 +108,39 @@ pub struct PBFTOrderProtocol<D, NT, PL, RP>
     // Require any synchronization
     message_log: Log<D, PL>,
     // The proposer of this replica
-    proposer: Arc<Proposer<D, NT, RP>>,
+    proposer: Arc<Proposer<D, NT>>,
     // The networking layer for a Node in the network (either Client or Replica)
     node: Arc<NT>,
 
     executor: ExecutorHandle<D>,
 }
 
-impl<D, NT, PL, RP> Orderable for PBFTOrderProtocol<D, NT, PL, RP>
+impl<D, NT, PL> Orderable for PBFTOrderProtocol<D, NT, PL>
     where D: 'static + ApplicationData,
-          NT: 'static + OrderProtocolSendNode<D, PBFT<D, RP>>,
-          RP: 'static + ReconfigurationProtocolMessage,
+          NT: 'static + OrderProtocolSendNode<D, PBFT<D>>,
           PL: Clone {
     fn sequence_number(&self) -> SeqNo {
         self.consensus.sequence_number()
     }
 }
 
-impl<D, NT, PL, RP> OrderProtocolTolerance for PBFTOrderProtocol<D, NT, PL, RP>
+impl<D, NT, PL> OrderProtocolTolerance for PBFTOrderProtocol<D, NT, PL>
     where D: 'static + ApplicationData,
-          NT: 'static + OrderProtocolSendNode<D, PBFT<D, RP>>,
-          PL: Clone,
-          RP: 'static + ReconfigurationProtocolMessage {
+          NT: 'static + OrderProtocolSendNode<D, PBFT<D>>,
+          PL: Clone, {
     fn get_n_for_f(f: usize) -> usize {
         3 * f + 1
     }
 }
 
-impl<D, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL, RP>
+impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
-          RP: ReconfigurationProtocolMessage + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
           PL: Clone {
-    type Serialization = PBFTConsensus<D, RP>;
-    type Config = PBFTConfig<D, RP>;
+    type Serialization = PBFTConsensus<D>;
+    type Config = PBFTConfig<D>;
 
-    fn initialize(config: PBFTConfig<D, RP>, args: OrderingProtocolArgs<D, NT, PL>) -> Result<Self> where
+    fn initialize(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT, PL>) -> Result<Self> where
         Self: Sized,
     {
         Self::initialize_protocol(config, args, None)
@@ -153,8 +150,8 @@ impl<D, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL,
         self.synchronizer.view()
     }
 
-    fn handle_off_ctx_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request, QuorumJoinCert<RP>>>>)
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    fn handle_off_ctx_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>)
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let (header, message) = message.into_inner();
 
         match message.into_inner() {
@@ -189,8 +186,8 @@ impl<D, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL,
         Ok(())
     }
 
-    fn poll(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request, QuorumJoinCert<RP>>, D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    fn poll(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         trace!("{:?} // Polling {:?}", self.node.id(), self.phase);
 
         match self.phase {
@@ -203,8 +200,8 @@ impl<D, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL,
         }
     }
 
-    fn process_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request, QuorumJoinCert<RP>>>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    fn process_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         match self.phase {
             ConsensusPhase::NormalPhase => {
                 self.update_normal_phase(message)
@@ -235,15 +232,14 @@ impl<D, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL,
     }
 
     fn handle_timeout(&mut self, timeout: Vec<RqTimeout>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         if self.consensus.is_catching_up() {
             warn!("{:?} // Ignoring timeouts while catching up", self.node.id());
 
             return Ok(OrderProtocolExecResult::Success);
         }
 
-        let status = self.synchronizer
-            .client_requests_timed_out(&self.synchronizer, self.node.id(), &timeout);
+        let status = self.synchronizer.client_requests_timed_out(self.node.id(), &timeout);
 
         match status {
             SynchronizerStatus::RequestsTimedOut { forwarded, stopped } => {
@@ -275,12 +271,11 @@ impl<D, NT, PL, RP> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL,
     }
 }
 
-impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
+impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
           PL: Clone {
-    fn initialize_protocol(config: PBFTConfig<D, RP>, args: OrderingProtocolArgs<D, NT, PL>,
+    fn initialize_protocol(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT, PL>,
                            initial_state: Option<DecisionLog<D::Request>>) -> Result<Self> {
         let PBFTConfig {
             node_id,
@@ -297,15 +292,15 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
 
         let consensus_guard = ProposerConsensusGuard::new(view.clone(), watermark);
 
-        let consensus = Consensus::<D, PL, RP>::new_replica(node_id, &sync.view(), executor.clone(),
-                                                            SeqNo::ZERO, watermark, consensus_guard.clone(),
-                                                            timeouts.clone(), persistent_log.clone());
+        let consensus = Consensus::<D, PL>::new_replica(node_id, &sync.view(), executor.clone(),
+                                                        SeqNo::ZERO, watermark, consensus_guard.clone(),
+                                                        timeouts.clone(), persistent_log.clone());
 
         let dec_log = initialize_decided_log::<D, PL>(node_id, persistent_log, initial_state)?;
 
-        let proposer = Proposer::<D, NT, RP>::new(node.clone(), batch_input, sync.clone(), timeouts.clone(),
-                                                  executor.clone(), consensus_guard.clone(),
-                                                  proposer_config);
+        let proposer = Proposer::<D, NT>::new(node.clone(), batch_input, sync.clone(), timeouts.clone(),
+                                              executor.clone(), consensus_guard.clone(),
+                                              proposer_config);
 
         let replica = Self {
             phase: ConsensusPhase::NormalPhase,
@@ -340,8 +335,8 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
         Ok(replica)
     }
 
-    fn poll_sync_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request, QuorumJoinCert<RP>>, D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    fn poll_sync_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
 
         // retrieve a view change message to be processed
         let poll_result = self.synchronizer.poll();
@@ -370,8 +365,8 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
         }
     }
 
-    fn poll_normal_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request, QuorumJoinCert<RP>>, D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    fn poll_normal_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         // check if we have STOP messages to be processed,
         // and update our phase when we start installing
         // the new view
@@ -423,8 +418,8 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
         }
     }
 
-    fn update_sync_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request, QuorumJoinCert<RP>>>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    fn update_sync_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let (header, protocol) = message.into_inner();
 
         match protocol.into_inner() {
@@ -463,8 +458,8 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
         Ok(OrderProtocolExecResult::Success)
     }
 
-    fn update_normal_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request, QuorumJoinCert<RP>>>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    fn update_normal_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let (header, protocol) = message.into_inner();
 
         match protocol.into_inner() {
@@ -507,7 +502,7 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
         header: Header,
         message: ConsensusMessage<D::Request>,
     ) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let seq = self.consensus.sequence_number();
 
         // debug!(
@@ -574,8 +569,8 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
 
     /// Advance the sync phase of the algorithm
     fn adv_sync(&mut self, header: Header,
-                message: ViewChangeMessage<D::Request, QuorumJoinCert<RP>>) -> SyncPhaseRes<D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+                message: ViewChangeMessage<D::Request>) -> SyncPhaseRes<D::Request>
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let status = self.synchronizer.process_message(
             header,
             message,
@@ -626,12 +621,11 @@ impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
     }
 }
 
-impl<D, NT, PL, RP> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL, RP>
+impl<D, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
           PL: Clone {
-    type StateSerialization = PBFTConsensus<D, RP>;
+    type StateSerialization = PBFTConsensus<D>;
 
     fn initialize_with_initial_state(config: Self::Config,
                                      args: OrderingProtocolArgs<D, NT, PL>,
@@ -642,7 +636,7 @@ impl<D, NT, PL, RP> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT
     fn install_state(&mut self,
                      view_info: View<Self::Serialization>,
                      dec_log: DecLog<Self::StateSerialization>) -> Result<Vec<D::Request>>
-        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D, RP>, PBFTConsensus<D, RP>> {
+        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
         info!("{:?} // Installing decision log with Seq No {:?} and View {:?}", self.node.id(),
                 dec_log.sequence_number(), view_info);
 
@@ -689,10 +683,9 @@ impl<D, NT, PL, RP> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT
     }
 }
 
-impl<D, NT, PL, RP> PBFTOrderProtocol<D, NT, PL, RP>
+impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
           PL: Clone {
     pub(crate) fn switch_phase(&mut self, new_phase: ConsensusPhase) {
         info!("{:?} // Switching from phase {:?} to phase {:?}", self.node.id(), self.phase, new_phase);
@@ -752,10 +745,9 @@ const CF_PRE_PREPARES: &str = "PRE_PREPARES";
 const CF_PREPARES: &str = "PREPARES";
 const CF_COMMIT: &str = "COMMITS";
 
-impl<D, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTConsensus<D, RP>> for PBFTOrderProtocol<D, NT, PL, RP>
+impl<D, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus<D>> for PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D, RP>>, PL: Clone {
+          NT: OrderProtocolSendNode<D, PBFT<D>>, PL: Clone {
     fn message_types() -> Vec<&'static str> {
         vec![
             CF_PRE_PREPARES,
@@ -764,7 +756,7 @@ impl<D, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTConsensus
         ]
     }
 
-    fn get_type_for_message(msg: &LoggableMessage<PBFTConsensus<D, RP>>) -> Result<&'static str> {
+    fn get_type_for_message(msg: &LoggableMessage<PBFTConsensus<D>>) -> Result<&'static str> {
         match msg.kind() {
             ConsensusMessageKind::PrePrepare(_) => {
                 Ok(CF_PRE_PREPARES)
@@ -778,7 +770,7 @@ impl<D, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTConsensus
         }
     }
 
-    fn init_proof_from(metadata: SerProofMetadata<PBFTConsensus<D, RP>>, messages: Vec<StoredMessage<LoggableMessage<PBFTConsensus<D, RP>>>>) -> SerProof<PBFTConsensus<D, RP>> {
+    fn init_proof_from(metadata: SerProofMetadata<PBFTConsensus<D>>, messages: Vec<StoredMessage<LoggableMessage<PBFTConsensus<D>>>>) -> SerProof<PBFTConsensus<D>> {
         let mut pre_prepares = Vec::with_capacity(messages.len() / 2);
         let mut prepares = Vec::with_capacity(messages.len() / 2);
         let mut commits = Vec::with_capacity(messages.len() / 2);
@@ -800,11 +792,11 @@ impl<D, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTConsensus
         Proof::new(metadata, pre_prepares, prepares, commits)
     }
 
-    fn init_dec_log(proofs: Vec<SerProof<PBFTConsensus<D, RP>>>) -> DecLog<PBFTConsensus<D, RP>> {
+    fn init_dec_log(proofs: Vec<SerProof<PBFTConsensus<D>>>) -> DecLog<PBFTConsensus<D>> {
         DecisionLog::from_proofs(proofs)
     }
 
-    fn decompose_proof(proof: &SerProof<PBFTConsensus<D, RP>>) -> (&SerProofMetadata<PBFTConsensus<D, RP>>, Vec<&StoredMessage<LoggableMessage<PBFTConsensus<D, RP>>>>) {
+    fn decompose_proof(proof: &SerProof<PBFTConsensus<D>>) -> (&SerProofMetadata<PBFTConsensus<D>>, Vec<&StoredMessage<LoggableMessage<PBFTConsensus<D>>>>) {
         let mut messages = Vec::new();
 
         for message in proof.pre_prepares() {
@@ -822,19 +814,36 @@ impl<D, NT, PL, RP> PersistableOrderProtocol<PBFTConsensus<D, RP>, PBFTConsensus
         (proof.metadata(), messages)
     }
 
-    fn decompose_dec_log(proofs: &DecLog<PBFTConsensus<D, RP>>) -> Vec<&SerProof<PBFTConsensus<D, RP>>> {
+    fn decompose_dec_log(proofs: &DecLog<PBFTConsensus<D>>) -> Vec<&SerProof<PBFTConsensus<D>>> {
         proofs.proofs().iter().collect()
     }
 }
 
-impl<D, NT, PL, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, NT, PL, RP>
+impl<D, NT, PL, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
-          PL: Clone {
-    fn attempt_network_view_change(&mut self, join_certificate: QuorumJoinCert<RP>) -> Result<ReconfigurationAttemptResult> {
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
+          PL: OrderingProtocolLog<PBFTConsensus<D>> {
+    fn attempt_quorum_node_join(&mut self, joining_node: NodeId) -> Result<ReconfigurationAttemptResult> {
         self.switch_phase(ConsensusPhase::SyncPhase);
 
-        Ok(self.synchronizer.attempt_join_quorum(join_certificate, &*self.node, &self.timeouts))
+        Ok(self.synchronizer.start_join_quorum(joining_node, &*self.node, &self.timeouts, &self.message_log))
+    }
+
+    fn joining_quorum(&mut self) -> Result<ReconfigurationAttemptResult> {
+        let result = self.synchronizer.attempt_join_quorum(&*self.node, &self.timeouts);
+
+        match &result {
+            ReconfigurationAttemptResult::Failed => {
+                warn!("Failed to join quorum")
+            }
+            ReconfigurationAttemptResult::AlreadyPartOfQuorum => {}
+            ReconfigurationAttemptResult::InProgress => {
+                self.switch_phase(ConsensusPhase::SyncPhase);
+            }
+            _ => {}
+        }
+
+        Ok(result)
     }
 }
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index a0a3697b..939fffc3 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -52,9 +52,8 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// Read the current state, if existent, from the persistent storage
     ///
     /// FIXME: The view initialization might have to be changed if we want to introduce reconfiguration
-    pub fn read_current_state<RP>(&self, n: usize, f: usize) -> Result<Option<(ViewInfo, DecisionLog<D::Request>)>>
-        where RP: ReconfigurationProtocolMessage + 'static,
-              PL: StatefulOrderingProtocolLog<PBFTConsensus<D, RP>, PBFTConsensus<D, RP>> {
+    pub fn read_current_state(&self, n: usize, f: usize) -> Result<Option<(ViewInfo, DecisionLog<D::Request>)>>
+        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
         let option = self.persistent_log.read_state(OperationMode::BlockingSync)?;
 
         if let Some((view, dec_log)) = option {
@@ -77,12 +76,10 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// We can use this method when we want to prevent a clone, as this takes
     /// just a reference.
     /// This is mostly used for pre prepares as they contain all the requests and are therefore very expensive to send
-    pub fn insert_consensus<RP>(
+    pub fn insert_consensus(
         &mut self,
         consensus_msg: StoredConsensusMessage<D::Request>,
-    ) where
-        RP: ReconfigurationProtocolMessage + 'static,
-        PL: OrderingProtocolLog<PBFTConsensus<D, RP>>,
+    ) where PL: OrderingProtocolLog<PBFTConsensus<D>>,
     {
         if let Err(err) = self
             .persistent_log
@@ -96,10 +93,8 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// This is done when we receive the final SYNC message from the leader
     /// which contains all of the collects
     /// If we are missing the request determined by the
-    pub fn install_proof<RP>(&mut self, seq: SeqNo, proof: Proof<D::Request>) -> Result<ProtocolConsensusDecision<D::Request>>
-        where
-            RP: ReconfigurationProtocolMessage + 'static,
-            PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    pub fn install_proof(&mut self, seq: SeqNo, proof: Proof<D::Request>) -> Result<ProtocolConsensusDecision<D::Request>>
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let batch_execution_info = ProtocolConsensusDecision::from(&proof);
 
         if let Some(decision) = self.decision_log().last_decision() {
@@ -123,20 +118,16 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     }
 
     /// Clear the occurrences of a seq no from the decision log
-    pub fn clear_last_occurrence<RP>(&mut self, seq: SeqNo)
-        where
-            RP: ReconfigurationProtocolMessage + 'static,
-            PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    pub fn clear_last_occurrence(&mut self, seq: SeqNo)
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         if let Err(err) = self.persistent_log.write_invalidate(OperationMode::NonBlockingSync(None), seq) {
             error!("Failed to invalidate last occurrence {:?}", err);
         }
     }
 
     /// Update the log state, received from the CST protocol.
-    pub fn install_state<RP>(&mut self, view: ViewInfo, dec_log: DecisionLog<D::Request>)
-        where
-            RP: ReconfigurationProtocolMessage + 'static,
-            PL: StatefulOrderingProtocolLog<PBFTConsensus<D, RP>, PBFTConsensus<D, RP>> {
+    pub fn install_state(&mut self, view: ViewInfo, dec_log: DecisionLog<D::Request>)
+        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
 
         //Replace the log
         self.dec_log = dec_log.clone();
@@ -171,9 +162,8 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
 
     /// Register that all the batches for a given decision have already been received
     /// Basically persists the metadata for a given consensus num
-    pub fn all_batches_received<RP>(&mut self, metadata: ProofMetadata)
-        where RP: ReconfigurationProtocolMessage + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    pub fn all_batches_received(&mut self, metadata: ProofMetadata)
+        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
         self.persistent_log.write_proof_metadata(OperationMode::NonBlockingSync(None),
                                                  metadata).unwrap();
     }
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index f5e9d305..fc4ca0c6 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -37,14 +37,13 @@ pub type BatchType<R> = Vec<StoredRequestMessage<R>>;
 ///Handles taking requests from the client pools and storing the requests in the log,
 ///as well as creating new batches and delivering them to the batch_channel
 ///Another thread will then take from this channel and propose the requests
-pub struct Proposer<D, NT, RP>
-    where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static {
+pub struct Proposer<D, NT>
+    where D: ApplicationData + 'static, {
     /// Channel for the reception of batches from the pre processing module
     batch_reception: BatchOutput<D::Request>,
     /// Network Node
     node_ref: Arc<NT>,
-    synchronizer: Arc<Synchronizer<D, RP>>,
+    synchronizer: Arc<Synchronizer<D>>,
     timeouts: Timeouts,
     consensus_guard: Arc<ProposerConsensusGuard>,
     // Should we shut down?
@@ -78,13 +77,12 @@ impl<D> ProposeBuilder<D> where D: ApplicationData {
 ///The size of the batch channel
 const BATCH_CHANNEL_SIZE: usize = 128;
 
-impl<D, NT, RP> Proposer<D, NT, RP>
-    where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static {
+impl<D, NT> Proposer<D, NT>
+    where D: ApplicationData + 'static, {
     pub fn new(
         node: Arc<NT>,
         batch_input: BatchOutput<D::Request>,
-        sync: Arc<Synchronizer<D, RP>>,
+        sync: Arc<Synchronizer<D>>,
         timeouts: Timeouts,
         executor_handle: ExecutorHandle<D>,
         consensus_guard: Arc<ProposerConsensusGuard>,
@@ -110,7 +108,7 @@ impl<D, NT, RP> Proposer<D, NT, RP>
 
     ///Start this work
     pub fn start(self: Arc<Self>) -> JoinHandle<()>
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static {
+        where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
         std::thread::Builder::new()
             .name(format!("Proposer thread"))
             .spawn(move || {
@@ -248,7 +246,7 @@ impl<D, NT, RP> Proposer<D, NT, RP>
     fn propose_unordered(
         &self,
         propose: &mut ProposeBuilder<D>,
-    ) -> bool where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
+    ) -> bool where NT: OrderProtocolSendNode<D, PBFT<D>> {
         if !propose.currently_accumulated.is_empty() {
             let current_batch_size = propose.currently_accumulated.len();
 
@@ -316,7 +314,7 @@ impl<D, NT, RP> Proposer<D, NT, RP>
     /// Returns true if a batch was proposed
     fn propose_ordered(&self, is_leader: bool,
                        propose: &mut ProposeBuilder<D>, ) -> bool
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
+        where NT: OrderProtocolSendNode<D, PBFT<D>> {
 
         //Now let's deal with ordered requests
         if is_leader {
@@ -373,8 +371,7 @@ impl<D, NT, RP> Proposer<D, NT, RP>
         seq: SeqNo,
         view: &ViewInfo,
         mut currently_accumulated: Vec<StoredRequestMessage<D::Request>>,
-    ) where
-        NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
+    ) where NT: OrderProtocolSendNode<D, PBFT<D>> {
         let has_pending_messages = self.consensus_guard.has_pending_view_change_reqs();
 
         let is_view_change_empty = {
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index d30d772c..34cd4996 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -19,6 +19,7 @@ use atlas_common::crypto::hash::Digest;
 use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_pop_message, tbo_queue_message};
+use atlas_common::threadpool::join;
 use atlas_communication::message::{Header, StoredMessage, WireMessage};
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
@@ -26,7 +27,6 @@ use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::ReconfigurationAttemptResult;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_core::reconfiguration_protocol::QuorumJoinCert;
 use atlas_core::request_pre_processing::RequestPreProcessor;
 use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
@@ -51,12 +51,12 @@ pub mod view;
 /// The code provided in the first argument gets executed
 /// The first T is the type of message that we should expect to be returned from the queue
 macro_rules! extract_msg {
-    ($t:ty, $t2:ty => $g:expr, $q:expr) => {
-        extract_msg!($t, $t2 => {}, $g, $q)
+    ($t:ty => $g:expr, $q:expr) => {
+        extract_msg!($t => {}, $g, $q)
     };
 
-    ($t:ty, $t2:ty => $opt:block, $g:expr, $q:expr) => {
-        if let Some(stored) = tbo_pop_message::<StoredMessage<ViewChangeMessage<$t, $t2>>>($q) {
+    ($t:ty => $opt:block, $g:expr, $q:expr) => {
+        if let Some(stored) = tbo_pop_message::<StoredMessage<ViewChangeMessage<$t>>>($q) {
             $opt
             let (header, message) = stored.into_inner();
             SynchronizerPollStatus::NextMessage(header, message)
@@ -113,21 +113,21 @@ macro_rules! finalize_view_change {
 /// of the view change protocol, as well as a value to be proposed in the `SYNC` message.
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
-pub struct LeaderCollects<O, JC> {
+pub struct LeaderCollects<O> {
     //The pre prepare message, created and signed by the leader to be executed when the view change is
     // Done
     proposed: FwdConsensusMessage<O>,
     // The collect messages the leader has received.
-    collects: Vec<StoredMessage<ViewChangeMessage<O, JC>>>,
+    collects: Vec<StoredMessage<ViewChangeMessage<O>>>,
 }
 
-impl<O, JC> LeaderCollects<O, JC> {
+impl<O> LeaderCollects<O> {
     /// Gives up ownership of the inner values of this `LeaderCollects`.
     pub fn into_inner(
         self,
     ) -> (
         FwdConsensusMessage<O>,
-        Vec<StoredMessage<ViewChangeMessage<O, JC>>>,
+        Vec<StoredMessage<ViewChangeMessage<O>>>,
     ) {
         (self.proposed, self.collects)
     }
@@ -170,37 +170,53 @@ impl Sound {
 }
 
 /// Represents a queue of view change messages that arrive out of context into a node.
-pub struct TboQueue<O, JC> {
+pub struct TboQueue<O> {
     // the current view
     view: ViewInfo,
+    // The preview to the next view, in case we are in the process of changing views
+    next_view: Option<ViewInfo>,
     // Stores the previous view, for useful information when changing views
     previous_view: Option<ViewInfo>,
     // probe messages from this queue instead of
     // fetching them from the network
     get_queue: bool,
-    // stores all Node Quorum Update messages for the next view
-    quorum_join: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O, JC>>>>,
     // stores all STOP messages for the next view
-    stop: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O, JC>>>>,
+    stop: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O>>>>,
     // stores all STOP-DATA messages for the next view
-    stop_data: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O, JC>>>>,
+    stop_data: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O>>>>,
     // stores all SYNC messages for the next view
-    sync: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O, JC>>>>,
+    sync: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O>>>>,
 }
 
-impl<O, JC> TboQueue<O, JC> {
+impl<O> TboQueue<O> {
     pub(crate) fn new(view: ViewInfo) -> Self {
         Self {
             view,
+            next_view: None,
             previous_view: None,
             get_queue: false,
-            quorum_join: VecDeque::new(),
             stop: VecDeque::new(),
             stop_data: VecDeque::new(),
             sync: VecDeque::new(),
         }
     }
 
+    pub fn install_next_view(&mut self, view: ViewInfo) {
+        self.next_view = Some(view);
+    }
+
+    pub fn next_view(&self) -> Option<&ViewInfo> {
+        self.next_view.as_ref()
+    }
+
+    /// Advance to the next view we are working on
+    pub fn advance(&mut self) -> bool {
+        if let Some(next_view) = self.next_view.take() {
+            self.install_view(next_view)
+        } else {
+            false
+        }
+    }
 
     /// Installs a new view into the queue.
     pub fn install_view(&mut self, view: ViewInfo) -> bool {
@@ -235,7 +251,6 @@ impl<O, JC> TboQueue<O, JC> {
     }
 
     fn next_instance_queue(&mut self) {
-        tbo_advance_message_queue(&mut self.quorum_join);
         tbo_advance_message_queue(&mut self.stop);
         tbo_advance_message_queue(&mut self.stop_data);
         tbo_advance_message_queue(&mut self.sync);
@@ -243,32 +258,23 @@ impl<O, JC> TboQueue<O, JC> {
 
     /// Queues a view change message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    pub fn queue(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
+    pub fn queue(&mut self, h: Header, m: ViewChangeMessage<O>) {
         match m.kind() {
-            ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) => self.queue_stop(h, m),
+            ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_) => self.queue_stop(h, m),
             ViewChangeMessageKind::StopData(_) => self.queue_stop_data(h, m),
             ViewChangeMessageKind::Sync(_) => self.queue_sync(h, m),
-            ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
-                self.queue_node_quorum_join(h, m)
-            }
         }
     }
 
     /// Verifies if we have new `STOP` messages to be processed for
     /// the current view.
     pub fn can_process_stops(&self) -> bool {
-        self.stop.get(0).map(|deque| deque.len() > 0).unwrap_or(false) ||
-            self.quorum_join.get(0).map(|deque| deque.len() > 0).unwrap_or(false)
-    }
-
-    fn queue_node_quorum_join(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
-        let seq = self.view.sequence_number().next();
-        tbo_queue_message(seq, &mut self.quorum_join, StoredMessage::new(h, m))
+        self.stop.get(0).map(|deque| deque.len() > 0).unwrap_or(false)
     }
 
     /// Queues a `STOP` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    fn queue_stop(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
+    fn queue_stop(&mut self, h: Header, m: ViewChangeMessage<O>) {
         // NOTE: we use next() because we want to retrieve messages
         // for v+1, as we haven't started installing the new view yet
         let seq = self.view.sequence_number().next();
@@ -277,15 +283,15 @@ impl<O, JC> TboQueue<O, JC> {
 
     /// Queues a `STOP-DATA` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    fn queue_stop_data(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
-        let seq = self.view.sequence_number();
+    fn queue_stop_data(&mut self, h: Header, m: ViewChangeMessage<O>) {
+        let seq = self.view.sequence_number().next();
         tbo_queue_message(seq, &mut self.stop_data, StoredMessage::new(h, m))
     }
 
     /// Queues a `SYNC` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    fn queue_sync(&mut self, h: Header, m: ViewChangeMessage<O, JC>) {
-        let seq = self.view.sequence_number();
+    fn queue_sync(&mut self, h: Header, m: ViewChangeMessage<O>) {
+        let seq = self.view.sequence_number().next();
         tbo_queue_message(seq, &mut self.sync, StoredMessage::new(h, m))
     }
 
@@ -357,21 +363,21 @@ pub enum SynchronizerStatus<O> {
 
 /// Represents the status of calling `poll()` on a `Synchronizer`.
 #[derive(Clone)]
-pub enum SynchronizerPollStatus<O, JC> {
+pub enum SynchronizerPollStatus<O> {
     /// The `Replica` associated with this `Synchronizer` should
     /// poll its network channel for more messages, as it has no messages
     /// That can be processed in cache
     Recv,
     /// A new view change message is available to be processed, retrieved from the
     /// Ordered queue
-    NextMessage(Header, ViewChangeMessage<O, JC>),
+    NextMessage(Header, ViewChangeMessage<O>),
     /// We need to resume the view change protocol, after
     /// running the CST protocol.
     ResumeViewChange,
 }
 
 ///A trait describing some of the necessary methods for the synchronizer
-pub trait AbstractSynchronizer<D, RP> where D: ApplicationData + 'static, RP: ReconfigurationProtocolMessage + 'static {
+pub trait AbstractSynchronizer<D> where D: ApplicationData + 'static {
     /// Returns information regarding the current view, such as
     /// the number of faulty replicas the system can tolerate.
     fn view(&self) -> ViewInfo;
@@ -380,25 +386,25 @@ pub trait AbstractSynchronizer<D, RP> where D: ApplicationData + 'static, RP: Re
     /// running the view change protocol.
     fn install_view(&self, view: ViewInfo);
 
-    fn queue(&self, header: Header, message: ViewChangeMessage<D::Request, QuorumJoinCert<RP>>);
+    fn queue(&self, header: Header, message: ViewChangeMessage<D::Request>);
 }
 
-type CollectsType<D: ApplicationData, JC> = IntMap<StoredMessage<ViewChangeMessage<D::Request, JC>>>;
+type CollectsType<D: ApplicationData> = IntMap<StoredMessage<ViewChangeMessage<D::Request>>>;
 
 ///The synchronizer for the SMR protocol
 /// This part of the protocol is responsible for handling the changing of views and
 /// for keeping track of any timed out client requests
-pub struct Synchronizer<D: ApplicationData, RP: ReconfigurationProtocolMessage> {
+pub struct Synchronizer<D: ApplicationData> {
     phase: Cell<ProtoPhase>,
     //Tbo queue, keeps track of the current view and keeps messages arriving in order
-    tbo: Mutex<TboQueue<D::Request, QuorumJoinCert<RP>>>,
+    tbo: Mutex<TboQueue<D::Request>>,
     //Stores currently received requests from other nodes
     stopped: RefCell<IntMap<Vec<StoredRequestMessage<D::Request>>>>,
     //Stores currently received requests from other nodes
-    currently_adding_node: Cell<Option<(NodeId, QuorumJoinCert<RP>)>>,
+    currently_adding_node: Cell<Option<NodeId>>,
     //TODO: This does not require a Mutex I believe since it's only accessed when
     // Processing messages (which is always done in the replica thread)
-    collects: Mutex<CollectsType<D, QuorumJoinCert<RP>>>,
+    collects: Mutex<CollectsType<D>>,
     // Used to store the finalize state when we are forced to run the CST protocol
     finalize_state: RefCell<Option<FinalizeState<D::Request>>>,
     // We need to keep track of whether we are entering the quorum
@@ -413,11 +419,10 @@ pub struct Synchronizer<D: ApplicationData, RP: ReconfigurationProtocolMessage>
 /// So we protect collects, watching and tbo as those are the fields that are going to be
 /// accessed by both those threads.
 /// Since the other fields are going to be accessed by just 1 thread, we just need them to be Send, which they are
-unsafe impl<D: ApplicationData, RP: ReconfigurationProtocolMessage> Sync for Synchronizer<D, RP> {}
+unsafe impl<D: ApplicationData> Sync for Synchronizer<D> {}
 
-impl<D, RP> AbstractSynchronizer<D, RP> for Synchronizer<D, RP>
-    where D: ApplicationData + 'static,
-          RP: ReconfigurationProtocolMessage + 'static {
+impl<D> AbstractSynchronizer<D> for Synchronizer<D>
+    where D: ApplicationData + 'static, {
     /// Returns some information regarding the current view, such as
     /// the number of faulty replicas the system can tolerate.
     fn view(&self) -> ViewInfo {
@@ -438,15 +443,12 @@ impl<D, RP> AbstractSynchronizer<D, RP> for Synchronizer<D, RP>
         }
     }
 
-    fn queue(&self, header: Header, message: ViewChangeMessage<D::Request, QuorumJoinCert<RP>>) {
+    fn queue(&self, header: Header, message: ViewChangeMessage<D::Request>) {
         self.tbo.lock().unwrap().queue(header, message)
     }
 }
 
-impl<D, RP> Synchronizer<D, RP>
-    where
-        D: ApplicationData + 'static,
-        RP: ReconfigurationProtocolMessage + 'static
+impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 {
     pub fn new_follower(view: ViewInfo) -> Arc<Self> {
         Arc::new(Self {
@@ -494,8 +496,18 @@ impl<D, RP> Synchronizer<D, RP>
         }))
     }
 
+    /// The next view that is going to be processed
+    fn next_view(&self) -> Option<ViewInfo> { self.tbo.lock().unwrap().next_view().cloned() }
+
+    /// The previous view that was processed
     fn previous_view(&self) -> Option<ViewInfo> { self.tbo.lock().unwrap().previous_view().clone() }
 
+    /// Install the next view which we are currently working on changing to
+    fn install_next_view(&self, view: ViewInfo) { self.tbo.lock().unwrap().install_next_view(view) }
+
+    /// Advance the view the next one in the queue
+    fn advance_view(&self) -> bool { self.tbo.lock().unwrap().advance() }
+
     /// Signal this `TboQueue` that it may be able to extract new
     /// view change messages from its internal storage.
     pub fn signal(&self) {
@@ -511,13 +523,13 @@ impl<D, RP> Synchronizer<D, RP>
     /// Check if we can process new view change messages.
     /// If there are pending messages that are now processable (but weren't when we received them)
     /// We return them. If there are no pending messages then we will wait for new messages from other replicas
-    pub fn poll(&self) -> SynchronizerPollStatus<D::Request, QuorumJoinCert<RP>> {
+    pub fn poll(&self) -> SynchronizerPollStatus<D::Request> {
         let mut tbo_guard = self.tbo.lock().unwrap();
         match self.phase.get() {
             _ if !tbo_guard.get_queue => SynchronizerPollStatus::Recv,
             ProtoPhase::Init => {
                 //If we are in the init phase and there is a pending request, move to the stopping phase
-                let result = extract_msg!(D::Request, QuorumJoinCert<RP>  =>
+                let result = extract_msg!(D::Request  =>
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop
                 );
@@ -525,7 +537,7 @@ impl<D, RP> Synchronizer<D, RP>
                 match &result {
                     SynchronizerPollStatus::NextMessage(h, vc) => {
                         match vc.kind() {
-                            ViewChangeMessageKind::StopQuorumJoin(_, _) => {
+                            ViewChangeMessageKind::StopQuorumJoin(_) => {
                                 self.phase.replace(ProtoPhase::ViewStopping(0));
                             }
                             _ => {
@@ -536,48 +548,22 @@ impl<D, RP> Synchronizer<D, RP>
                     _ => {}
                 };
 
-                // Reset the tbo guard so we also query the join cert queue
-                tbo_guard.get_queue = true;
-
-                match result {
-                    SynchronizerPollStatus::Recv => {
-                        // if we don't have any pending stop messages, check the join quorum
-                        extract_msg!(D::Request, QuorumJoinCert<RP> => {self.phase.replace(ProtoPhase::ViewStopping(0));},
-                            &mut tbo_guard.get_queue, &mut tbo_guard.quorum_join)
-                    }
-                    _ => {
-                        result
-                    }
-                }
+                result
             }
             ProtoPhase::Stopping(_) | ProtoPhase::Stopping2(_) | ProtoPhase::ViewStopping(_) | ProtoPhase::ViewStopping2(_) => {
-                let result = extract_msg!(D::Request, QuorumJoinCert<RP> =>
+                extract_msg!(D::Request=>
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop
-                );
-
-                // Reset the tbo guard so we also query the join cert queue
-                tbo_guard.get_queue = true;
-
-                match result {
-                    SynchronizerPollStatus::Recv => {
-                        // if we don't have any pending stop messages, check the join quorum
-                        extract_msg!(D::Request, QuorumJoinCert<RP> =>
-                            &mut tbo_guard.get_queue, &mut tbo_guard.quorum_join)
-                    }
-                    _ => {
-                        result
-                    }
-                }
+                )
             }
             ProtoPhase::StoppingData(_) => {
-                extract_msg!(D::Request, QuorumJoinCert<RP>  =>
+                extract_msg!(D::Request  =>
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.stop_data
                 )
             }
             ProtoPhase::Syncing => {
-                extract_msg!(D::Request, QuorumJoinCert<RP>  =>
+                extract_msg!(D::Request  =>
                     &mut tbo_guard.get_queue,
                     &mut tbo_guard.sync
                 )
@@ -592,15 +578,15 @@ impl<D, RP> Synchronizer<D, RP>
     pub fn process_message<NT, PL>(
         &self,
         header: Header,
-        message: ViewChangeMessage<D::Request, QuorumJoinCert<RP>>,
+        message: ViewChangeMessage<D::Request>,
         timeouts: &Timeouts,
         log: &mut Log<D, PL>,
         rq_pre_processor: &RequestPreProcessor<D::Request>,
-        consensus: &mut Consensus<D, PL, RP>,
+        consensus: &mut Consensus<D, PL>,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
+        where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         debug!("{:?} // Processing view change message {:?} in phase {:?} from {:?}",
                node.id(),
@@ -611,16 +597,7 @@ impl<D, RP> Synchronizer<D, RP>
         match self.phase.get() {
             ProtoPhase::Init => {
                 return match message.kind() {
-                    ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
-                        let mut guard = self.tbo.lock().unwrap();
-
-                        debug!("{:?} // Received {:?} message while in init state. Queueing",  node.id(), message);
-
-                        guard.queue_node_quorum_join(header, message);
-
-                        SynchronizerStatus::Nil
-                    }
-                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) => {
+                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_) => {
                         let mut guard = self.tbo.lock().unwrap();
 
                         debug!("{:?} // Received {:?} message while in init state. Queueing", node.id(), message);
@@ -662,16 +639,7 @@ impl<D, RP> Synchronizer<D, RP>
                 let next_seq = current_view.sequence_number().next();
 
                 let i = match message.kind() {
-                    ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
-                        let mut guard = self.tbo.lock().unwrap();
-
-                        guard.queue_node_quorum_join(header, message);
-
-                        debug!("{:?} // Received node join message while in stopping state. Queueing", node.id());
-
-                        return stop_status!(i, &current_view);
-                    }
-                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) if msg_seq != next_seq => {
+                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_) if msg_seq != next_seq => {
                         debug!("{:?} // Received stop message {:?} that does not match up to our local view {:?}", node.id(), message, current_view);
 
                         let mut guard = self.tbo.lock().unwrap();
@@ -686,7 +654,7 @@ impl<D, RP> Synchronizer<D, RP>
                         // drop attempts to vote twice
                         return stop_status!(i, &current_view);
                     }
-                    ViewChangeMessageKind::StopQuorumJoin(_, _) => {
+                    ViewChangeMessageKind::StopQuorumJoin(_) => {
                         warn!("{:?} // Received stop quorum join message while in stopping state. Ignoring", node.id());
 
                         return stop_status!(i, &current_view);
@@ -739,6 +707,7 @@ impl<D, RP> Synchronizer<D, RP>
                 if let ProtoPhase::Stopping(_i) = self.phase.get() {
                     return if i > current_view.params().f() {
                         self.begin_view_change(None, &**node, timeouts, log);
+
                         SynchronizerStatus::Running
                     } else {
                         self.phase.replace(ProtoPhase::Stopping(i));
@@ -759,7 +728,7 @@ impl<D, RP> Synchronizer<D, RP>
 
                     warn!("{:?} // Stopping quorum reached, moving to next view {:?}. ", node.id(), next_view);
 
-                    self.install_view(next_view);
+                    self.install_next_view(next_view);
 
                     match &self.accessory {
                         SynchronizerAccessory::Replica(rep) => {
@@ -788,26 +757,8 @@ impl<D, RP> Synchronizer<D, RP>
                 let current_view = self.view();
                 let next_seq = current_view.sequence_number().next();
 
-                let (received, node_id, jc) = match message.kind() {
-                    ViewChangeMessageKind::NodeQuorumJoin(node_id, join_certificate) if msg_seq == next_seq => {
-                        //TODO: Verify node join certificate
-
-                        self.currently_adding_node.replace(Some((*node_id, join_certificate.clone())));
-
-                        self.begin_quorum_view_change(Some((*node_id, join_certificate.clone())), &**node, timeouts, log);
-
-                        return SynchronizerStatus::Running;
-                    }
-                    ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
-                        let mut guard = self.tbo.lock().unwrap();
-
-                        guard.queue_node_quorum_join(header, message);
-
-                        debug!("{:?} // Received node join message while in stopping state. Queueing", node.id());
-
-                        return stop_status!(received, &current_view);
-                    }
-                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) if msg_seq != next_seq => {
+                let (received, node_id) = match message.kind() {
+                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_) if msg_seq != next_seq => {
                         debug!("{:?} // Received stop message {:?} that does not match up to our local view {:?}", node.id(), message, current_view);
 
                         let mut guard = self.tbo.lock().unwrap();
@@ -827,12 +778,12 @@ impl<D, RP> Synchronizer<D, RP>
 
                         return SynchronizerStatus::Running;
                     }
-                    ViewChangeMessageKind::StopQuorumJoin(node, join_cert) if self.currently_adding_node.get().is_some() && self.currently_adding_node.get().unwrap() != *node => {
-                        error!("{:?} // Received stop quorum join message with node that does not match the current received ", node.id());
+                    ViewChangeMessageKind::StopQuorumJoin(node) if self.currently_adding_node.get().is_some() && self.currently_adding_node.get().unwrap() != *node => {
+                        error!("{:?} // Received stop quorum join message with node that does not match the current received", node.id());
 
                         return SynchronizerStatus::Running;
                     }
-                    ViewChangeMessageKind::StopQuorumJoin(node, join_cert) => (received + 1, *node, join_cert),
+                    ViewChangeMessageKind::StopQuorumJoin(node) => (received + 1, *node),
                     ViewChangeMessageKind::StopData(_) => {
                         match &self.accessory {
                             SynchronizerAccessory::Follower(_) => {
@@ -869,8 +820,6 @@ impl<D, RP> Synchronizer<D, RP>
 
                 if let ProtoPhase::ViewStopping(_received) = self.phase.get() {
                     return if received > current_view.params().f() {
-                        self.currently_adding_node.replace(Some((node_id, jc.clone())));
-
                         self.begin_quorum_view_change(None, &**node, timeouts, log);
 
                         SynchronizerStatus::Running
@@ -882,7 +831,9 @@ impl<D, RP> Synchronizer<D, RP>
                 }
 
                 if received >= current_view.params().quorum() {
-                    let next_view = current_view.next_view_with_new_node(self.currently_adding_node.get().unwrap().0);
+                    self.currently_adding_node.replace(Some(node_id));
+
+                    let next_view = current_view.next_view_with_new_node(self.currently_adding_node.get().unwrap());
 
                     let previous_view = current_view.clone();
 
@@ -893,7 +844,7 @@ impl<D, RP> Synchronizer<D, RP>
 
                     warn!("{:?} // Stopping quorum reached, moving to next view {:?}. ", node.id(), next_view);
 
-                    self.install_view(next_view);
+                    self.install_next_view(next_view);
 
                     match &self.accessory {
                         SynchronizerAccessory::Replica(rep) => {
@@ -928,25 +879,14 @@ impl<D, RP> Synchronizer<D, RP>
                         //Obtain the view seq no of the message
                         let msg_seq = message.sequence_number();
 
-                        let current_view = self.view();
-                        let seq = current_view.sequence_number();
+                        let next_view = self.next_view().expect("We can't be here without having installed the next view?");
+                        let seq = next_view.sequence_number();
 
                         // reject STOP-DATA messages if we are not the leader
                         let mut collects_guard = self.collects.lock().unwrap();
 
                         let i = match message.kind() {
-                            ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
-                                {
-                                    let mut guard = self.tbo.lock().unwrap();
-
-                                    guard.queue_node_quorum_join(header, message);
-
-                                    debug!("{:?} // Received stop message while in stopping data state. Queueing", node.id());
-                                }
-
-                                return SynchronizerStatus::Running;
-                            }
-                            ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) => {
+                            ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_) => {
                                 {
                                     let mut guard = self.tbo.lock().unwrap();
 
@@ -961,7 +901,7 @@ impl<D, RP> Synchronizer<D, RP>
                                 warn!("{:?} // Received stop data message for view {:?} but we are in view {:?}",
                                       node.id(), msg_seq, seq);
 
-                                if current_view.peek(msg_seq).leader() == node.id() {
+                                if next_view.peek(msg_seq).leader() == node.id() {
                                     warn!("{:?} // We are the leader of the view of the received message, so we will accept it",
                                       node.id());
 
@@ -977,8 +917,7 @@ impl<D, RP> Synchronizer<D, RP>
 
                                 return SynchronizerStatus::Running;
                             }
-                            ViewChangeMessageKind::StopData(_)
-                            if current_view.leader() != node.id() =>
+                            ViewChangeMessageKind::StopData(_) if next_view.leader() != node.id() =>
                                 {
                                     warn!("{:?} // Received stop data message but we are not the leader of the current view",
                                       node.id());
@@ -1020,10 +959,9 @@ impl<D, RP> Synchronizer<D, RP>
                         // the new leader isn't forging messages.
 
                         // store collects from this STOP-DATA
-                        collects_guard
-                            .insert(header.from().into(), StoredMessage::new(header, message));
+                        collects_guard.insert(header.from().into(), StoredMessage::new(header, message));
 
-                        if i < current_view.params().quorum() {
+                        if i < next_view.params().quorum() {
                             self.phase.replace(ProtoPhase::StoppingData(i));
 
                             return SynchronizerStatus::Running;
@@ -1035,11 +973,11 @@ impl<D, RP> Synchronizer<D, RP>
                             //   STOP-DATA proofs so other replicas
                             //   can repeat the leader's computation
 
-                            let previous_view = self.previous_view();
+                            let current_view = self.view();
 
                             //Since all of these requests were done in the previous view of the algorithm
                             // then we should also use the previous view to verify the validity of them
-                            let previous_view_ref = previous_view.as_ref().unwrap_or(&current_view);
+                            let previous_view_ref = &current_view;
 
                             let proof = Self::highest_proof(&*collects_guard,
                                                             previous_view_ref, &**node);
@@ -1057,7 +995,7 @@ impl<D, RP> Synchronizer<D, RP>
                             let normalized_collects: Vec<Option<&CollectData<D::Request>>> =
                                 Self::normalized_collects(&*collects_guard, curr_cid).collect();
 
-                            let sound = sound(&current_view, &normalized_collects);
+                            let sound = sound(&next_view, &normalized_collects);
 
                             if !sound.test() {
                                 //FIXME: BFT-SMaRt doesn't do anything if `sound`
@@ -1082,7 +1020,7 @@ impl<D, RP> Synchronizer<D, RP>
                             let (header, message) = {
                                 info!("{:?} // Forged pre-prepare: {}", node.id(), p.len());
 
-                                let forged_pre_prepare = consensus.forge_propose(p, self);
+                                let forged_pre_prepare = consensus.forge_propose(p, &next_view);
 
                                 let (message, digest) = node.serialize_digest_message(forged_pre_prepare).unwrap();
 
@@ -1093,8 +1031,8 @@ impl<D, RP> Synchronizer<D, RP>
                                 //Create the pre-prepare message that contains the requests
                                 //Collected during the STOPPING DATA phase
                                 let (h, _) = WireMessage::new(
-                                    self.view().leader(),
-                                    node.id(),
+                                    next_view.leader(),
+                                    next_view.leader(),
                                     buf,
                                     prng_state.next_state(),
                                     Some(digest),
@@ -1117,17 +1055,17 @@ impl<D, RP> Synchronizer<D, RP>
                                 .cloned().collect();
 
                             let message = PBFTMessage::ViewChange(ViewChangeMessage::new(
-                                current_view.sequence_number(),
+                                next_view.sequence_number(),
                                 ViewChangeMessageKind::Sync(LeaderCollects {
                                     proposed: fwd_request.clone(),
                                     collects,
                                 }),
                             ));
 
-                            let node_id = node.id();
+                            let our_id = node.id();
 
-                            let targets = current_view.quorum_members().clone().into_iter()
-                                .filter(move |&id| id != node_id);
+                            let targets = next_view.quorum_members().clone().into_iter()
+                                .filter(move |&id| id != our_id);
 
                             node.broadcast(message, targets);
 
@@ -1154,23 +1092,12 @@ impl<D, RP> Synchronizer<D, RP>
             }
             ProtoPhase::Syncing => {
                 let msg_seq = message.sequence_number();
-                let current_view = self.view();
-                let seq = current_view.sequence_number();
+                let next_view = self.next_view().expect("");
+                let seq = next_view.sequence_number();
 
                 // reject SYNC messages if these were not sent by the leader
                 let (proposed, collects) = match message.kind() {
-                    ViewChangeMessageKind::NodeQuorumJoin(_, _) => {
-                        {
-                            let mut guard = self.tbo.lock().unwrap();
-
-                            guard.queue_node_quorum_join(header, message);
-
-                            debug!("{:?} // Received node join message while in syncing state. Queueing", node.id());
-                        }
-
-                        return SynchronizerStatus::Running;
-                    }
-                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_, _) => {
+                    ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_) => {
                         {
                             let mut guard = self.tbo.lock().unwrap();
 
@@ -1205,7 +1132,7 @@ impl<D, RP> Synchronizer<D, RP>
                     }
                     ViewChangeMessageKind::Sync(_) if msg_seq != seq => {
                         {
-                            debug!("{:?} // Received sync message whose sequence number does not match our current one {:?} vs {:?}. Queueing", node.id(), message, current_view);
+                            debug!("{:?} // Received sync message whose sequence number does not match our current one {:?} vs {:?}. Queueing", node.id(), message, next_view);
 
                             let mut guard = self.tbo.lock().unwrap();
 
@@ -1214,7 +1141,7 @@ impl<D, RP> Synchronizer<D, RP>
 
                         return SynchronizerStatus::Running;
                     }
-                    ViewChangeMessageKind::Sync(_) if header.from() != current_view.leader() => {
+                    ViewChangeMessageKind::Sync(_) if header.from() != next_view.leader() => {
                         //You're not the leader, what are you saying
                         return SynchronizerStatus::Running;
                     }
@@ -1227,9 +1154,9 @@ impl<D, RP> Synchronizer<D, RP>
 
                 // leader has already performed this computation in the
                 // STOP-DATA phase of Mod-SMaRt
-                let signed: Vec<_> = signed_collects::<D, _, _>(&**node, collects);
+                let signed: Vec<_> = signed_collects::<D, _>(&**node, collects);
 
-                let proof = highest_proof::<D, _, _, _>(&current_view, &**node, signed.iter());
+                let proof = highest_proof::<D, _, _>(&next_view, &**node, signed.iter());
 
                 let curr_cid = proof
                     .map(|p| p.sequence_number())
@@ -1239,7 +1166,7 @@ impl<D, RP> Synchronizer<D, RP>
                 let normalized_collects: Vec<_> =
                     { normalized_collects(curr_cid, collect_data(signed.iter())).collect() };
 
-                let sound = sound(&current_view, &normalized_collects);
+                let sound = sound(&next_view, &normalized_collects);
 
                 if !sound.test() {
                     error!("{:?} // The view change is not sound. Cancelling.", node.id());
@@ -1281,11 +1208,11 @@ impl<D, RP> Synchronizer<D, RP>
         &self,
         log: &mut Log<D, PL>,
         timeouts: &Timeouts,
-        consensus: &mut Consensus<D, PL, RP>,
+        consensus: &mut Consensus<D, PL>,
         node: &Arc<NT>,
     ) -> Option<()>
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
+        where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         let state = self.finalize_state.borrow_mut().take()?;
 
@@ -1306,14 +1233,73 @@ impl<D, RP> Synchronizer<D, RP>
         Some(())
     }
 
+    /// Start the quorum join procedure to integrate the given joining node into the current quorum
+    /// of the system
+    pub fn start_join_quorum<NT, PL>(&self, joining_node: NodeId, node: &NT, timeouts: &Timeouts, log: &Log<D, PL>) -> ReconfigurationAttemptResult
+        where NT: OrderProtocolSendNode<D, PBFT<D>>,
+              PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        let current_view = self.view();
+
+        info!("{:?} // Starting the quorum join procedure for node {:?}", node.id(), joining_node);
+
+        if current_view.quorum_members().contains(&joining_node) {
+            //We are already a part of the quorum, so we don't need to do anything
+            info!("{:?} // Attempted to add node {:?} quorum but it is already a part of the quorum", node.id(), joining_node);
+
+            return ReconfigurationAttemptResult::AlreadyPartOfQuorum;
+        }
+
+        if joining_node == node.id() {
+            unreachable!("We should never try to add ourselves to the quorum, there is a specific function for that")
+        } else {
+            self.begin_quorum_view_change(Some(joining_node), node, timeouts, log);
+        }
+
+        return ReconfigurationAttemptResult::InProgress;
+    }
+
+    /// Prepare ourselves for the quorum join procedure by stopping the current view and starting a new one
+    pub fn attempt_join_quorum<NT>(&self, node: &NT,
+                                   timeouts: &Timeouts) -> ReconfigurationAttemptResult
+        where NT: OrderProtocolSendNode<D, PBFT<D>>, {
+
+        let current_view = self.view();
+
+        info!("{:?} // Attempting to join quorum, skipping stop phase as we are not yet part of the quorum", node.id());
+
+        if current_view.quorum_members().contains(&node.id()) {
+            //We are already a part of the quorum, so we don't need to do anything
+            info!("{:?} // Attempted to join quorum, but we are already a part of it", node.id());
+            return ReconfigurationAttemptResult::AlreadyPartOfQuorum;
+        }
+
+        // Simulate that we were accepted into the quorum
+        let view = current_view.next_view_with_new_node(node.id());
+
+        self.entering_quorum.replace(true);
+
+        self.install_next_view(view.clone());
+
+        if view.leader() == node.id() {
+            // If we are the leader of the next view, then we should move to the stopping data phase and wait
+            // For the rest of the nodes to send us the information
+            self.phase.replace(ProtoPhase::StoppingData(0));
+        } else {
+            self.phase.replace(ProtoPhase::Syncing);
+        }
+
+        return ReconfigurationAttemptResult::InProgress;
+    }
+
+    /// Trigger a view change locally
     pub fn begin_quorum_view_change<NT, PL>(
         &self,
-        join_cert: Option<(NodeId, QuorumJoinCert<RP>)>,
+        join_cert: Option<NodeId>,
         node: &NT,
         timeouts: &Timeouts,
         _log: &Log<D, PL>, )
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>>,
-              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
+        where NT: OrderProtocolSendNode<D, PBFT<D>>,
+              PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         debug!("Beginning quorum view change with certificate {} at phase {:?}",  join_cert.is_some(), self.phase.get());
 
@@ -1332,55 +1318,25 @@ impl<D, RP> Synchronizer<D, RP>
                 return;
             }
             _ => {
-                self.currently_adding_node.replace(None);
+                self.stopped.borrow_mut().clear();
+                self.collects.lock().unwrap().clear();
+
+                self.phase.replace(ProtoPhase::ViewStopping2(0));
             }
         }
 
         match &self.accessory {
             SynchronizerAccessory::Follower(_) => {}
             SynchronizerAccessory::Replica(replica) => {
-                replica.handle_begin_quorum_view_change(self, timeouts, node, join_cert)
-            }
-        }
-    }
-
-    pub fn attempt_join_quorum<NT>(&self, join_certificate: QuorumJoinCert<RP>,
-                                   node: &NT,
-                                   timeouts: &Timeouts) -> ReconfigurationAttemptResult
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>>, {
-        let current_view = self.view();
-
-        info!("{:?} // Attempting to join quorum", node.id());
+                if let Some(join_cert) = join_cert {
+                    self.currently_adding_node.replace(Some(join_cert));
 
-        if current_view.quorum_members().contains(&node.id()) {
-            //We are already a part of the quorum, so we don't need to do anything
-            info!("{:?} // Attempted to join quorum, but we are already a part of it", node.id());
-            return ReconfigurationAttemptResult::AlreadyPartOfQuorum;
-        }
-
-        let message = ViewChangeMessage::new(
-            current_view.sequence_number().next(),
-            ViewChangeMessageKind::NodeQuorumJoin(node.id(), join_certificate),
-        );
-
-        node.broadcast_signed(PBFTMessage::ViewChange(message), current_view.quorum_members().clone().into_iter());
-
-        // Simulate that we were accepted into the quorum
-        let view = current_view.next_view_with_new_node(node.id());
-
-        if view.leader() == node.id() {
-            // If we are the leader of the next view, then we should move to the stopping data phase and wait
-            // For the rest of the nodes to send us the information
-            self.phase.replace(ProtoPhase::StoppingData(0));
-        } else {
-            self.phase.replace(ProtoPhase::Syncing);
+                    // We only want to send our STOP message when we have received the notification
+                    // From the reconfiguration protocol, even if there are already f+1 STOP messages
+                    replica.handle_begin_quorum_view_change(self, timeouts, node, join_cert)
+                }
+            }
         }
-
-        self.entering_quorum.replace(true);
-
-        self.install_view(view);
-
-        return ReconfigurationAttemptResult::InProgress;
     }
 
     /// Trigger a view change locally.
@@ -1395,9 +1351,8 @@ impl<D, RP> Synchronizer<D, RP>
         node: &NT,
         timeouts: &Timeouts,
         _log: &Log<D, PL>,
-    )
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>>,
-              PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
+    ) where NT: OrderProtocolSendNode<D, PBFT<D>>,
+            PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         match (self.phase.get(), &timed_out) {
             // we have received STOP messages from peer nodes,
@@ -1406,11 +1361,13 @@ impl<D, RP> Synchronizer<D, RP>
             // when `timed_out` is `None`, we were called from `process_message`,
             // so we need to update our phase with a new received message
             (ProtoPhase::Stopping(i), None) => {
+                // We update here to Stopping2 as we will send our own message right after this
                 self.phase.replace(ProtoPhase::Stopping2(i + 1));
             }
             //When the timeout is not null, this means it was called from timed out client requests
             //And therefore we don't increase the received message count, just update the phase to Stopping2
             (ProtoPhase::Stopping(i), _) => {
+                // We have begun our own stopping protocol, so we will update our phase to Stopping2
                 self.phase.replace(ProtoPhase::Stopping2(i));
             }
             (ProtoPhase::ViewStopping(_), None) | (ProtoPhase::ViewStopping2(_), None) => {
@@ -1463,7 +1420,7 @@ impl<D, RP> Synchronizer<D, RP>
         _normalized_collects: Vec<Option<&CollectData<D::Request>>>,
         log: &Log<D, PL>,
     ) -> FinalizeStatus<D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
+        where PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         let last_executed_cid = proof.as_ref().map(|p| p.sequence_number()).unwrap_or(SeqNo::ZERO);
 
@@ -1494,12 +1451,11 @@ impl<D, RP> Synchronizer<D, RP>
         state: FinalizeState<D::Request>,
         log: &mut Log<D, PL>,
         timeouts: &Timeouts,
-        consensus: &mut Consensus<D, PL, RP>,
+        consensus: &mut Consensus<D, PL>,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
-        where
-            NT: OrderProtocolSendNode<D, PBFT<D, RP>> + 'static,
-            PL: OrderingProtocolLog<PBFTConsensus<D, RP>>
+        where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
+              PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
         let FinalizeState {
             curr_cid,
@@ -1508,6 +1464,9 @@ impl<D, RP> Synchronizer<D, RP>
             last_proof
         } = state;
 
+        // Finalize the view change by advancing our tbo queue to the new view
+        self.advance_view();
+
         let view = self.view();
 
         warn!("{:?} // Finalizing view change to view {:?} and consensus ID {:?}", node.id(), view, curr_cid);
@@ -1542,11 +1501,9 @@ impl<D, RP> Synchronizer<D, RP>
         };
 
         // finalize view change by broadcasting a PREPARE msg
-        consensus.finalize_view_change((header, message), self, timeouts, log, node);
+        consensus.finalize_view_change((header, message), &view, self, timeouts, log, node);
 
-        // skip queued messages from the current view change
-        // and update proto phase
-        self.tbo.lock().unwrap().next_instance_queue();
+        // Update proto phase
         self.phase.replace(ProtoPhase::Init);
 
         if self.entering_quorum.get() && view.quorum_members().contains(&node.id()) {
@@ -1591,7 +1548,7 @@ impl<D, RP> Synchronizer<D, RP>
     pub fn forward_requests<NT>(&self,
                                 timed_out: Vec<StoredRequestMessage<D::Request>>,
                                 node: &NT)
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
+        where NT: OrderProtocolSendNode<D, PBFT<D>> {
         match &self.accessory {
             SynchronizerAccessory::Follower(_) => {}
             SynchronizerAccessory::Replica(rep) => {
@@ -1604,7 +1561,6 @@ impl<D, RP> Synchronizer<D, RP>
     /// Requests that have timed out
     pub fn client_requests_timed_out(
         &self,
-        base_sync: &Synchronizer<D, RP>,
         my_id: NodeId,
         seq: &Vec<RqTimeout>,
     ) -> SynchronizerStatus<D::Request> {
@@ -1613,7 +1569,7 @@ impl<D, RP> Synchronizer<D, RP>
                 SynchronizerStatus::Nil
             }
             SynchronizerAccessory::Replica(rep) => {
-                rep.client_requests_timed_out(base_sync, my_id, seq)
+                rep.client_requests_timed_out(self, my_id, seq)
             }
         }
     }
@@ -1624,7 +1580,7 @@ impl<D, RP> Synchronizer<D, RP>
 // may be executing the same CID when there is a leader change
     #[inline]
     fn normalized_collects<'a>(
-        collects: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
+        collects: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request>>>,
         in_exec: SeqNo,
     ) -> impl Iterator<Item=Option<&'a CollectData<D::Request>>> {
         let values = collects.values();
@@ -1637,13 +1593,13 @@ impl<D, RP> Synchronizer<D, RP>
     // TODO: quorum sizes may differ when we implement reconfiguration
     #[inline]
     fn highest_proof<'a, NT>(
-        guard: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
+        guard: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request>>>,
         view: &ViewInfo,
         node: &NT,
     ) -> Option<&'a Proof<D::Request>>
-        where NT: OrderProtocolSendNode<D, PBFT<D, RP>>
+        where NT: OrderProtocolSendNode<D, PBFT<D>>
     {
-        highest_proof::<D, _, _, _>(&view, node, guard.values())
+        highest_proof::<D, _, _>(&view, node, guard.values())
     }
 }
 
@@ -1846,8 +1802,8 @@ fn certified_value<O>(
     count > curr_view.params().f()
 }
 
-fn collect_data<'a, O: 'a, JC: 'a>(
-    collects: impl Iterator<Item=&'a StoredMessage<ViewChangeMessage<O, JC>>>,
+fn collect_data<'a, O: 'a>(
+    collects: impl Iterator<Item=&'a StoredMessage<ViewChangeMessage<O>>>,
 ) -> impl Iterator<Item=&'a CollectData<O>> {
     collects.filter_map(|stored| match stored.message().kind() {
         ViewChangeMessageKind::StopData(collects) => Some(collects),
@@ -1868,26 +1824,23 @@ fn normalized_collects<'a, O: 'a>(
     })
 }
 
-fn signed_collects<D, NT, RP>(
+fn signed_collects<D, NT>(
     node: &NT,
-    collects: Vec<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
-) -> Vec<StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>
-    where
-        D: ApplicationData + 'static,
-        RP: ReconfigurationProtocolMessage + 'static,
-        NT: OrderProtocolSendNode<D, PBFT<D, RP>>
+    collects: Vec<StoredMessage<ViewChangeMessage<D::Request>>>,
+) -> Vec<StoredMessage<ViewChangeMessage<D::Request>>>
+    where D: ApplicationData + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D>>
 {
     collects
         .into_iter()
-        .filter(|stored| validate_signature::<D, _, _, _>(node, stored))
+        .filter(|stored| validate_signature::<D, _, _>(node, stored))
         .collect()
 }
 
-fn validate_signature<'a, D, M, NT, RP>(node: &'a NT, stored: &'a StoredMessage<M>) -> bool
+fn validate_signature<'a, D, M, NT>(node: &'a NT, stored: &'a StoredMessage<M>) -> bool
     where
         D: ApplicationData + 'static,
-        RP: ReconfigurationProtocolMessage + 'static,
-        NT: OrderProtocolSendNode<D, PBFT<D, RP>>
+        NT: OrderProtocolSendNode<D, PBFT<D>>
 {
     //TODO: Fix this as I believe it will always be false
     let wm = match WireMessage::from_header(*stored.header()) {
@@ -1913,16 +1866,15 @@ fn validate_signature<'a, D, M, NT, RP>(node: &'a NT, stored: &'a StoredMessage<
     wm.is_valid(Some(&key), false)
 }
 
-fn highest_proof<'a, D, I, NT, RP>(
+fn highest_proof<'a, D, I, NT>(
     view: &ViewInfo,
     node: &NT,
     collects: I,
 ) -> Option<&'a Proof<D::Request>>
     where
         D: ApplicationData + 'static,
-        I: Iterator<Item=&'a StoredMessage<ViewChangeMessage<D::Request, QuorumJoinCert<RP>>>>,
-        RP: ReconfigurationProtocolMessage + 'static,
-        NT: OrderProtocolSendNode<D, PBFT<D, RP>>
+        I: Iterator<Item=&'a StoredMessage<ViewChangeMessage<D::Request>>>,
+        NT: OrderProtocolSendNode<D, PBFT<D>>
 {
     collect_data(collects)
         // fetch proofs
@@ -1942,7 +1894,7 @@ fn highest_proof<'a, D, I, NT, RP>(
                         .unwrap_or(false)
                 })
                 .filter(move |&stored|
-                    { validate_signature::<D, _, _, _>(node, stored) })
+                    { validate_signature::<D, _, _>(node, stored) })
                 .count() >= view.params().quorum();
 
             let prepares_valid = proof
@@ -1956,7 +1908,7 @@ fn highest_proof<'a, D, I, NT, RP>(
                         .unwrap_or(false)
                 })
                 .filter(move |&stored|
-                    { validate_signature::<D, _, _, _>(node, stored) })
+                    { validate_signature::<D, _, _>(node, stored) })
                 .count() >= view.params().quorum();
 
             debug!("{:?} // Proof {:?} is valid? commits valid: {:?} &&  prepares_valid: {:?}",
@@ -1968,7 +1920,7 @@ fn highest_proof<'a, D, I, NT, RP>(
 }
 
 
-impl<O, JC> Debug for SynchronizerPollStatus<O, JC> {
+impl<O> Debug for SynchronizerPollStatus<O> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
             SynchronizerPollStatus::Recv => {
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index edd4bb52..6d65d667 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -61,19 +61,17 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     ///
     /// Therefore, we start by clearing our stopped requests and treating them as
     /// newly proposed requests (by resetting their timer)
-    pub(super) fn handle_stopping_quorum<NT, PL, RP>(
+    pub(super) fn handle_stopping_quorum<NT, PL>(
         &self,
-        base_sync: &Synchronizer<D, RP>,
+        base_sync: &Synchronizer<D>,
         previous_view: ViewInfo,
-        consensus: &Consensus<D, PL, RP>,
+        consensus: &Consensus<D, PL>,
         log: &Log<D, PL>,
         pre_processor: &RequestPreProcessor<D::Request>,
         timeouts: &Timeouts,
         node: &NT,
-    )
-        where RP: ReconfigurationProtocolMessage + 'static,
-              NT: OrderProtocolSendNode<D, PBFT<D, RP>>,
-              PL: OrderingProtocolLog<PBFTConsensus<D, RP>> {
+    ) where NT: OrderProtocolSendNode<D, PBFT<D>>,
+            PL: OrderingProtocolLog<PBFTConsensus<D>> {
         // NOTE:
         // - install new view (i.e. update view seq no) (Done in the synchronizer)
         // - add requests from STOP into client requests
@@ -84,7 +82,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         self.take_stopped_requests_and_register_them(base_sync, pre_processor, timeouts);
         self.watch_all_requests(timeouts);
 
-        let view_info = base_sync.view();
+        let view_info = base_sync.next_view().expect("We should have a next view if we are at this point");
 
         let current_view_seq = view_info.sequence_number();
         let current_leader = view_info.leader();
@@ -112,14 +110,13 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     /// Start a new view change
     /// Receives the requests that it should send to the other
     /// nodes in its STOP message
-    pub(super) fn handle_begin_view_change<NT, RP>(
+    pub(super) fn handle_begin_view_change<NT>(
         &self,
-        base_sync: &Synchronizer<D, RP>,
+        base_sync: &Synchronizer<D>,
         timeouts: &Timeouts,
         node: &NT,
         timed_out: Option<Vec<StoredRequestMessage<D::Request>>>,
-    ) where RP: ReconfigurationProtocolMessage + 'static,
-            NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
+    ) where NT: OrderProtocolSendNode<D, PBFT<D>> {
         // stop all timers
         self.unwatch_all_requests(timeouts);
 
@@ -144,22 +141,18 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         node.broadcast(message, targets.into_iter());
     }
 
-    pub(super) fn handle_begin_quorum_view_change<NT, RP>(
+    pub(super) fn handle_begin_quorum_view_change<NT>(
         &self,
-        base_sync: &Synchronizer<D, RP>,
+        base_sync: &Synchronizer<D>,
         timeouts: &Timeouts,
         node: &NT,
-        join_cert: Option<(NodeId, QuorumJoinCert<RP>)>,
-    ) where RP: ReconfigurationProtocolMessage + 'static,
-            NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
-
+        join_cert: NodeId,
+    ) where NT: OrderProtocolSendNode<D, PBFT<D>> {
         let current_view = base_sync.view();
 
-        let (node_id, join_certificate) = self.joining_node(base_sync, join_cert);
-
-        info!("{:?} // Beginning a quorum view change to next view with new node: {:?}", node.id(), node_id);
+        info!("{:?} // Beginning a quorum view change to next view with new node: {:?}", node.id(), join_cert);
 
-        let message = ViewChangeMessageKind::StopQuorumJoin(node_id, join_certificate);
+        let message = ViewChangeMessageKind::StopQuorumJoin(join_cert);
 
         let message = ViewChangeMessage::new(current_view.sequence_number().next(), message);
 
@@ -232,10 +225,9 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     }
 
     /// Register all of the requests that are missing from the view change
-    fn take_stopped_requests_and_register_them<RP>(&self, base_sync: &Synchronizer<D, RP>,
-                                                   pre_processor: &RequestPreProcessor<D::Request>,
-                                                   timeouts: &Timeouts)
-        where RP: ReconfigurationProtocolMessage + 'static {
+    fn take_stopped_requests_and_register_them(&self, base_sync: &Synchronizer<D>,
+                                               pre_processor: &RequestPreProcessor<D::Request>,
+                                               timeouts: &Timeouts) {
         // TODO: maybe optimize this `stopped_requests` call, to avoid
         // a heap allocation of a `Vec`?
 
@@ -275,19 +267,12 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     /// Handle a timeout received from the timeouts layer.
     ///
     /// This timeout pertains to a group of client requests awaiting to be decided.
-    //
-    //
-    // TODO: fix current timeout impl, as most requests won't actually
-    // have surpassed their defined timeout period, after the timeout event
-    // is fired on the master channel of the core server task
-    //
-    pub fn client_requests_timed_out<RP>(
+    pub fn client_requests_timed_out(
         &self,
-        base_sync: &Synchronizer<D, RP>,
+        base_sync: &Synchronizer<D>,
         my_id: NodeId,
         timed_out_rqs: &Vec<RqTimeout>,
-    ) -> SynchronizerStatus<D::Request>
-        where RP: ReconfigurationProtocolMessage + 'static {
+    ) -> SynchronizerStatus<D::Request> {
 
         //// iterate over list of watched pending requests,
         //// and select the ones to be stopped or forwarded
@@ -355,15 +340,15 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         SynchronizerStatus::RequestsTimedOut { forwarded, stopped }
     }
 
+
     /// Forward the requests that timed out, `timed_out`, to all the nodes in the
     /// current view.
-    pub fn forward_requests<NT, RP>(
+    pub fn forward_requests<NT>(
         &self,
-        base_sync: &Synchronizer<D, RP>,
+        base_sync: &Synchronizer<D>,
         timed_out: Vec<StoredRequestMessage<D::Request>>,
         node: &NT,
-    ) where RP: ReconfigurationProtocolMessage + 'static,
-            NT: OrderProtocolSendNode<D, PBFT<D, RP>> {
+    ) where NT: OrderProtocolSendNode<D, PBFT<D>> {
         let message = ForwardedRequestsMessage::new(timed_out);
         let view = base_sync.view();
 
@@ -376,12 +361,11 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     /// to other nodes
     ///
     /// Clones all the nodes in the `stopped` list
-    fn stopped_requests<RP>(
+    fn stopped_requests(
         &self,
-        base_sync: &Synchronizer<D, RP>,
+        base_sync: &Synchronizer<D>,
         requests: Option<Vec<StoredRequestMessage<D::Request>>>,
-    ) -> Vec<StoredRequestMessage<D::Request>>
-        where RP: ReconfigurationProtocolMessage + 'static {
+    ) -> Vec<StoredRequestMessage<D::Request>> {
         // Use a hashmap so we are sure we don't send any repeat requests in our stop messages
         let mut all_reqs = collections::hash_map();
 
@@ -406,28 +390,11 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         all_reqs.drain().map(|(_, stop)| stop).collect()
     }
 
-    fn joining_node<RP>(
+    fn stopped_request_digests(
         &self,
-        base_sync: &Synchronizer<D, RP>,
-        join_cert: Option<(NodeId, QuorumJoinCert<RP>)>,
-    ) -> (NodeId, QuorumJoinCert<RP>)
-        where RP: ReconfigurationProtocolMessage + 'static {
-        let (node, join_cert) = match join_cert {
-            Some((node, join_cert)) => (node, join_cert),
-            None => {
-                let adding_node = base_sync.currently_adding_node.get();
-
-                adding_node.expect("We should have a node we are adding")
-            }
-        };
-    }
-
-    fn stopped_request_digests<RP>(
-        &self,
-        base_sync: &Synchronizer<D, RP>,
+        base_sync: &Synchronizer<D>,
         requests: Option<Vec<StoredRequestMessage<D::Request>>>,
-    ) -> Vec<ClientRqInfo>
-        where RP: ReconfigurationProtocolMessage + 'static {
+    ) -> Vec<ClientRqInfo> {
 
         // Use a hashmap so we are sure we don't send any repeat requests in our stop messages
         let mut all_reqs = collections::hash_set();
@@ -452,9 +419,8 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     }
 
     /// Drain our current received stopped messages
-    fn drain_stopped_request<RP>(&self, base_sync: &Synchronizer<D, RP>) ->
-    Vec<StoredRequestMessage<D::Request>>
-        where RP: ReconfigurationProtocolMessage + 'static {
+    fn drain_stopped_request(&self, base_sync: &Synchronizer<D>) ->
+    Vec<StoredRequestMessage<D::Request>> {
 
         // Use a hashmap so we are sure we don't send any repeat requests in our stop messages
         let mut all_reqs = collections::hash_map();

From 11afaf852232bac38d60f536eb302ad57f0d8d27 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 2 Aug 2023 19:08:04 +0100
Subject: [PATCH 31/80] Fixed issue with how the view installation from log
 transfers was handled by FeBFT due to a possible issue (frequent with large
 logs) when a new replica joined the network, which made it install the most
 recent view (with the newly entered node) and therefore ignore the SYNC
 request sent by the leader. Therefore we would also miss a pre prepare and
 wouldn't be able to do anything since we would always be missing that
 message. Fixed this by installing the previous version if there are any
 pending sync/stop data messages corresponding to the new view we are
 installing and then processing those messages until we organically reach the
 new view stage.

---
 febft-pbft-consensus/src/bft/mod.rs           |   9 +-
 febft-pbft-consensus/src/bft/sync/mod.rs      | 101 +++++++++++++++---
 febft-pbft-consensus/src/bft/sync/view/mod.rs |   9 ++
 3 files changed, 101 insertions(+), 18 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 2b880418..503f3b51 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -288,7 +288,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
                                  pre_processor, batch_input,
                                  node, persistent_log, quorum) = args;
 
-        let sync = Synchronizer::initialize_with_quorum(view.sequence_number(), quorum.clone(), timeout_dur)?;
+        let sync = Synchronizer::initialize_with_quorum(node_id, view.sequence_number(), quorum.clone(), timeout_dur)?;
 
         let consensus_guard = ProposerConsensusGuard::new(view.clone(), watermark);
 
@@ -391,7 +391,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
                             // As that has already been done by the adv sync method
                             return OrderProtocolPoll::RunCst;
                         }
-                        _ => unreachable!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message")
+                        _ => { warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message (Returned {:?})", result) }
                     }
                 } else {
                     // The synchronizer should never return anything other than a view
@@ -648,7 +648,10 @@ impl<D, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL
 
         info!("{:?} // Installing decision log with last execution {:?}", self.node.id(),last_exec);
 
-        self.synchronizer.install_view(view_info.clone());
+        if self.synchronizer.received_view_from_state_transfer(view_info.clone()) {
+            info!("{:?} // We have pending view state messages that we need to process. Switching to sync phase", self.node.id());
+            self.switch_phase(ConsensusPhase::SyncPhase);
+        }
 
         let start = Instant::now();
 
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 34cd4996..7070015c 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -219,11 +219,15 @@ impl<O> TboQueue<O> {
     }
 
     /// Installs a new view into the queue.
+    /// clear_tbo indicates if we should clear the index corresponding to the installed view
+    /// or not. (In the case of state transfers, for example we don't want to clear the index)
     pub fn install_view(&mut self, view: ViewInfo) -> bool {
         let index = view.sequence_number().index(self.view.sequence_number());
 
         return match index {
             Either::Right(i) if i > 0 => {
+                info!("Installing view {:?} into tbo queue, moving right by {}, currently at sequence number {:?}", view, i, self.view.sequence_number());
+
                 let prev_view = std::mem::replace(&mut self.view, view);
 
                 self.previous_view = Some(prev_view);
@@ -272,6 +276,18 @@ impl<O> TboQueue<O> {
         self.stop.get(0).map(|deque| deque.len() > 0).unwrap_or(false)
     }
 
+    /// Verifies if we have new `STOP` messages to be processed for
+    /// the current view.
+    pub fn can_process_stop_data(&self) -> bool {
+        self.stop_data.get(0).map(|deque| deque.len() > 0).unwrap_or(false)
+    }
+
+    /// Verifies if we have new `STOP` messages to be processed for
+    /// the current view.
+    pub fn can_process_sync(&self) -> bool {
+        self.sync.get(0).map(|deque| deque.len() > 0).unwrap_or(false)
+    }
+
     /// Queues a `STOP` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
     fn queue_stop(&mut self, h: Header, m: ViewChangeMessage<O>) {
@@ -384,7 +400,7 @@ pub trait AbstractSynchronizer<D> where D: ApplicationData + 'static {
 
     /// Install a new view received from the CST protocol, or from
     /// running the view change protocol.
-    fn install_view(&self, view: ViewInfo);
+    fn received_view_from_state_transfer(&self, view: ViewInfo) -> bool;
 
     fn queue(&self, header: Header, message: ViewChangeMessage<D::Request>);
 }
@@ -395,6 +411,8 @@ type CollectsType<D: ApplicationData> = IntMap<StoredMessage<ViewChangeMessage<D
 /// This part of the protocol is responsible for handling the changing of views and
 /// for keeping track of any timed out client requests
 pub struct Synchronizer<D: ApplicationData> {
+    node_id: NodeId,
+
     phase: Cell<ProtoPhase>,
     //Tbo queue, keeps track of the current view and keeps messages arriving in order
     tbo: Mutex<TboQueue<D::Request>>,
@@ -429,18 +447,58 @@ impl<D> AbstractSynchronizer<D> for Synchronizer<D>
         self.tbo.lock().unwrap().view().clone()
     }
 
-    /// Install a new view received from the CST protocol, or from
-    /// running the view change protocol.
-    fn install_view(&self, view: ViewInfo) {
+    /// Install a new view received from the CST protocol
+    /// This means that we are recovering from a failure or entering the
+    /// system, so we have to catch up to any new information about the view
+    /// Returns whether we should run the view change protocol to process pending
+    /// messages
+    fn received_view_from_state_transfer(&self, view: ViewInfo) -> bool {
         // FIXME: is the following line necessary?
-        let mut guard = self.tbo.lock().unwrap();
 
-        if !guard.install_view(view) {
-            // If we don't install a new view, then we don't want to forget our current state now do we?
+        let current_view = self.view();
+
+        if let Some(previous_view) = view.previous_view() {
+            if current_view.sequence_number() != previous_view.sequence_number() {
+                if !self.tbo.lock().unwrap().install_view(previous_view) {
+                    // If we don't install a new view, then we don't want to forget our current state now do we?
+
+                    debug!("Replacing our phase with Init");
+                    self.phase.replace(ProtoPhase::Init);
+                }
+            }
+
+            if view.leader() == self.node_id && self.can_process_stop_data() {
+                // If we are the leader of the new view, it means that we might still
+                // Have time to actually process the messages that we received from the
+                // Other nodes in time to maintain regency
+                self.phase.replace(ProtoPhase::StoppingData(0));
+                return true;
+            } else if self.can_process_sync() {
+                // If we are not the leader, then we need to check for the same thing
+                // But for sync messages. If we have a sync message in our queue,
+                // It means that we probably didn't receive the first decision after
+                // The view change in the log, which means we can't keep executing
+                // Without first processing this sync message
+                self.phase.replace(ProtoPhase::Syncing);
+
+                return true;
+            } else {
+                self.tbo.lock().unwrap().install_view(view);
+                return false;
+            }
+
+            self.install_next_view(view);
+        } else {
+            // This is the first view, so we can just install it
+            if !self.tbo.lock().unwrap().install_view(view) {
+                // If we don't install a new view, then we don't want to forget our current state now do we?
 
-            debug!("Replacing our phase with Init");
-            self.phase.replace(ProtoPhase::Init);
+                debug!("Replacing our phase with Init");
+                self.phase.replace(ProtoPhase::Init);
+            }
         }
+
+        false
     }
 
     fn queue(&self, header: Header, message: ViewChangeMessage<D::Request>) {
@@ -450,8 +508,9 @@ impl<D> AbstractSynchronizer<D> for Synchronizer<D>
 
 impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 {
-    pub fn new_follower(view: ViewInfo) -> Arc<Self> {
+    pub fn new_follower(node_id: NodeId, view: ViewInfo) -> Arc<Self> {
         Arc::new(Self {
+            node_id,
             phase: Cell::new(ProtoPhase::Init),
             stopped: RefCell::new(Default::default()),
             currently_adding_node: Cell::new(None),
@@ -463,8 +522,9 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         })
     }
 
-    pub fn new_replica(view: ViewInfo, timeout_dur: Duration) -> Arc<Self> {
+    pub fn new_replica(node_id: NodeId, view: ViewInfo, timeout_dur: Duration) -> Arc<Self> {
         Arc::new(Self {
+            node_id,
             phase: Cell::new(ProtoPhase::Init),
             stopped: RefCell::new(Default::default()),
             currently_adding_node: Cell::new(None),
@@ -477,7 +537,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     }
 
     /// Initialize a new `Synchronizer` with the given quorum members.
-    pub fn initialize_with_quorum(seq_no: SeqNo, quorum_members: Vec<NodeId>, timeout_dur: Duration) -> Result<Arc<Self>> {
+    pub fn initialize_with_quorum(node_id: NodeId, seq_no: SeqNo, quorum_members: Vec<NodeId>, timeout_dur: Duration) -> Result<Arc<Self>> {
         let n = quorum_members.len();
 
         let f = (n - 1) / 3;
@@ -485,6 +545,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         let view_info = ViewInfo::new(seq_no, n, f)?;
 
         Ok(Arc::new(Self {
+            node_id,
             phase: Cell::new(ProtoPhase::Init),
             tbo: Mutex::new(TboQueue::new(view_info)),
             stopped: RefCell::new(Default::default()),
@@ -520,6 +581,16 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         self.tbo.lock().unwrap().can_process_stops()
     }
 
+    /// Verifies if we have new `STOP_DATA` messages to be processed for
+    /// the next view.
+    pub fn can_process_stop_data(&self) -> bool {
+        self.tbo.lock().unwrap().can_process_stop_data()
+    }
+
+    /// Verifies if we have new `SYNC` messages to be processed for
+    /// the next view.
+    pub fn can_process_sync(&self) -> bool { self.tbo.lock().unwrap().can_process_sync() }
+
     /// Check if we can process new view change messages.
     /// If there are pending messages that are now processable (but weren't when we received them)
     /// We return them. If there are no pending messages then we will wait for new messages from other replicas
@@ -1092,7 +1163,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             }
             ProtoPhase::Syncing => {
                 let msg_seq = message.sequence_number();
-                let next_view = self.next_view().expect("");
+                let next_view = self.next_view().expect("We should have a next view in this situation");
                 let seq = next_view.sequence_number();
 
                 // reject SYNC messages if these were not sent by the leader
@@ -1262,14 +1333,14 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     pub fn attempt_join_quorum<NT>(&self, node: &NT,
                                    timeouts: &Timeouts) -> ReconfigurationAttemptResult
         where NT: OrderProtocolSendNode<D, PBFT<D>>, {
-
         let current_view = self.view();
 
         info!("{:?} // Attempting to join quorum, skipping stop phase as we are not yet part of the quorum", node.id());
 
         if current_view.quorum_members().contains(&node.id()) {
             //We are already a part of the quorum, so we don't need to do anything
-            info!("{:?} // Attempted to join quorum, but we are already a part of it", node.id());
+            info!("{:?} // Attempted to join quorum, but we are already a part of it, can we process sync messages {:?}", node.id(), self.can_process_sync());
+
             return ReconfigurationAttemptResult::AlreadyPartOfQuorum;
         }
 
diff --git a/febft-pbft-consensus/src/bft/sync/view/mod.rs b/febft-pbft-consensus/src/bft/sync/view/mod.rs
index e47e3595..27c9c05d 100644
--- a/febft-pbft-consensus/src/bft/sync/view/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/view/mod.rs
@@ -161,6 +161,15 @@ impl ViewInfo {
         Self::from_quorum(self.seq.next(), quorum_members).unwrap()
     }
 
+    pub fn previous_view(&self) -> Option<ViewInfo> {
+        if self.seq == SeqNo::ZERO {
+            return None;
+        }
+
+
+        Some(Self::new(self.seq.prev(), self.params.n(), self.params.f()).unwrap())
+    }
+
     /// Returns a new view with the specified sequence number.
     pub fn peek(&self, seq: SeqNo) -> ViewInfo {
         Self::new(seq, self.params.n(), self.params.f()).unwrap()

From 95dc51635e1d9103c9c724d6b129f9bc60a27ee1 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 2 Aug 2023 19:28:02 +0100
Subject: [PATCH 32/80] Fixed another bug when the second replica tries to
 join. Now getting a memory allocation error after the second replica joins.

---
 febft-pbft-consensus/src/bft/mod.rs      | 20 ++++++++++++++++----
 febft-pbft-consensus/src/bft/sync/mod.rs |  4 ++--
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 503f3b51..98d57f87 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -3,6 +3,7 @@
 //! By default, it is hidden to the user, unless explicitly enabled
 //! with the feature flag `expose_impl`.
 
+use std::fmt::{Debug, Formatter};
 use std::ops::Drop;
 use std::sync::Arc;
 use std::sync::atomic::AtomicBool;
@@ -380,18 +381,29 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
                 if let PBFTMessage::ViewChange(view_change) = message.into_inner() {
                     let result = self.adv_sync(header, view_change);
 
-                    match result {
+                    return match result {
                         SyncPhaseRes::RunSyncProtocol => {
                             self.switch_phase(ConsensusPhase::SyncPhase);
 
-                            return OrderProtocolPoll::RePoll;
+                            OrderProtocolPoll::RePoll
                         }
                         SyncPhaseRes::RunCSTProtocol => {
                             // We don't need to switch to the sync phase
                             // As that has already been done by the adv sync method
-                            return OrderProtocolPoll::RunCst;
+                            OrderProtocolPoll::RunCst
+                        }
+                        SyncPhaseRes::SyncProtocolNotNeeded => {
+                            warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, SyncProtocolNotNeeded");
+                            OrderProtocolPoll::RePoll
+                        }
+                        SyncPhaseRes::JoinedQuorum(_) => {
+                            warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, JoinedQuorum");
+                            OrderProtocolPoll::RePoll
+                        }
+                        SyncPhaseRes::SyncProtocolFinished(_) => {
+                            warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, Protocol Finished");
+                            OrderProtocolPoll::RePoll
                         }
-                        _ => { warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message (Returned {:?})", result) }
                     }
                 } else {
                     // The synchronizer should never return anything other than a view
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 7070015c..9433462e 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -467,12 +467,12 @@ impl<D> AbstractSynchronizer<D> for Synchronizer<D>
                 }
             }
 
+
             if view.leader() == self.node_id && self.can_process_stop_data() {
                 // If we are the leader of the new view, it means that we might still
                 // Have time to actually process the messages that we received from the
                 // Other nodes in time to maintain regency
                 self.phase.replace(ProtoPhase::StoppingData(0));
-                return true;
             } else if self.can_process_sync() {
                 // If we are not the leader, then we need to check for the same thing
                 // But for sync messages. If we have a sync message in our queue,
@@ -481,13 +481,13 @@ impl<D> AbstractSynchronizer<D> for Synchronizer<D>
                 // Without first processing this sync message
                 self.phase.replace(ProtoPhase::Syncing);
 
-                return true;
             } else {
                 self.tbo.lock().unwrap().install_view(view);
                 return false;
             }
 
             self.install_next_view(view);
+            return true;
         } else {
             // This is the first view, so we can just install it
             if !self.tbo.lock().unwrap().install_view(view) {

From 23d3c9fad26b755aa9aea468b40645427f78df18 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 2 Aug 2023 20:44:26 +0100
Subject: [PATCH 33/80] Fixed possible memory allocation issue related with
 transitioning connections between epoll workers (we weren't transitioning the
 read/write buffers, so they could be left halfway and cause missreads)

---
 febft-pbft-consensus/src/bft/sync/mod.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 9433462e..f452cd71 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -467,6 +467,8 @@ impl<D> AbstractSynchronizer<D> for Synchronizer<D>
                 }
             }
 
+            // Perform various check to assert we obtain all the necessary information to
+            // Recover from a fault
 
             if view.leader() == self.node_id && self.can_process_stop_data() {
                 // If we are the leader of the new view, it means that we might still

From 63175b53c262f2c4b243ba5f4254e698ebf8c232 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Fri, 4 Aug 2023 12:30:45 +0100
Subject: [PATCH 34/80] Fixed repeat joining of nodes in the quorum.

---
 febft-pbft-consensus/src/bft/mod.rs      | 14 ++++++++------
 febft-pbft-consensus/src/bft/sync/mod.rs | 16 +++++++++-------
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 98d57f87..6e95d006 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -79,7 +79,7 @@ pub enum SyncPhaseRes<O> {
     SyncProtocolNotNeeded,
     RunSyncProtocol,
     SyncProtocolFinished(Option<ProtocolConsensusDecision<O>>),
-    JoinedQuorum(Option<ProtocolConsensusDecision<O>>),
+    JoinedQuorum(Option<ProtocolConsensusDecision<O>>, NodeId),
     RunCSTProtocol,
 }
 
@@ -396,7 +396,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
                             warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, SyncProtocolNotNeeded");
                             OrderProtocolPoll::RePoll
                         }
-                        SyncPhaseRes::JoinedQuorum(_) => {
+                        SyncPhaseRes::JoinedQuorum(_, _) => {
                             warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, JoinedQuorum");
                             OrderProtocolPoll::RePoll
                         }
@@ -453,8 +453,10 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
                             }
                         }
                     }
-                    SyncPhaseRes::JoinedQuorum(to_execute) => {
-                        OrderProtocolExecResult::QuorumJoinResult(true, to_execute.map(|x| vec![x]))
+                    SyncPhaseRes::JoinedQuorum(to_execute, node) => {
+                        info!("Replica {:?} joined the quorum, with a decision to execute? {}", node, to_execute.is_some());
+
+                        OrderProtocolExecResult::QuorumJoined(to_execute.map(|x| vec![x]), node)
                     }
                     SyncPhaseRes::RunCSTProtocol => {
                         OrderProtocolExecResult::RunCst
@@ -618,12 +620,12 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
 
                 SyncPhaseRes::RunCSTProtocol
             }
-            SynchronizerStatus::NewViewJoinedQuorum(decision) => {
+            SynchronizerStatus::NewViewJoinedQuorum(decision, node) => {
                 //We have joined a quorum and we have a new view to execute
                 //We need to switch to the normal phase and execute the new view
                 self.switch_phase(ConsensusPhase::NormalPhase);
 
-                SyncPhaseRes::JoinedQuorum(decision)
+                SyncPhaseRes::JoinedQuorum(decision, node)
             }
             // should not happen...
             _ => {
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index f452cd71..8b8dc183 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -363,7 +363,7 @@ pub enum SynchronizerStatus<O> {
     NewView(Option<ProtocolConsensusDecision<O>>),
     /// The view change protocol just finished running and we
     /// have successfully joined the quorum.
-    NewViewJoinedQuorum(Option<ProtocolConsensusDecision<O>>),
+    NewViewJoinedQuorum(Option<ProtocolConsensusDecision<O>>, NodeId),
     /// Before we finish the view change protocol, we need
     /// to run the CST protocol.
     RunCst,
@@ -851,8 +851,8 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
                         return SynchronizerStatus::Running;
                     }
-                    ViewChangeMessageKind::StopQuorumJoin(node) if self.currently_adding_node.get().is_some() && self.currently_adding_node.get().unwrap() != *node => {
-                        error!("{:?} // Received stop quorum join message with node that does not match the current received", node.id());
+                    ViewChangeMessageKind::StopQuorumJoin(node_id) if self.currently_adding_node.get().is_some() && self.currently_adding_node.get().unwrap() != *node_id => {
+                        error!("{:?} // Received stop quorum join message with node that does not match the current received  {:?} vs {:?}", node.id(), self.currently_adding_node.get(), node_id);
 
                         return SynchronizerStatus::Running;
                     }
@@ -1350,6 +1350,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         let view = current_view.next_view_with_new_node(node.id());
 
         self.entering_quorum.replace(true);
+        self.currently_adding_node.replace(Some(self.node_id));
 
         self.install_next_view(view.clone());
 
@@ -1542,7 +1543,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
         let view = self.view();
 
-        warn!("{:?} // Finalizing view change to view {:?} and consensus ID {:?}", node.id(), view, curr_cid);
+        warn!("{:?} // Finalizing view change to view {:?} and consensus ID {:?}, Adding node? {:?}", node.id(), view, curr_cid, self.currently_adding_node.get());
 
         // we will get some value to be proposed because of the
         // check we did in `pre_finalize()`, guarding against no values
@@ -1579,9 +1580,10 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         // Update proto phase
         self.phase.replace(ProtoPhase::Init);
 
-        if self.entering_quorum.get() && view.quorum_members().contains(&node.id()) {
-            self.entering_quorum.replace(false);
-            SynchronizerStatus::NewViewJoinedQuorum(to_execute)
+        if self.currently_adding_node.get().is_some() {
+            let node = self.currently_adding_node.replace(None);
+
+            SynchronizerStatus::NewViewJoinedQuorum(to_execute, node.unwrap())
         } else {
             // resume normal phase
             SynchronizerStatus::NewView(to_execute)

From f67d593859c1259bf9f86b2e7f15fa830f791008 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Fri, 4 Aug 2023 15:50:06 +0100
Subject: [PATCH 35/80] Fixed a couple more issues with joining the quorum
 related to not iterating when stable, so not receiving quorum updates.

---
 febft-pbft-consensus/src/bft/sync/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 8b8dc183..038175bb 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -1323,7 +1323,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         }
 
         if joining_node == node.id() {
-            unreachable!("We should never try to add ourselves to the quorum, there is a specific function for that")
+            unreachable!("We should never try to add ourselves to the quorum this way, there is a specific function for that")
         } else {
             self.begin_quorum_view_change(Some(joining_node), node, timeouts, log);
         }

From 1c39c0249d081534b2c92387013f7d129ff247f1 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 10 Aug 2023 16:12:05 +0100
Subject: [PATCH 36/80] A lot of work on the new idea in reconfiguration
 protocol. (using voting to decide which node will join, instead of relying on
 the ordering protocol)

---
 febft-pbft-consensus/src/bft/mod.rs      |  67 ++++++---
 febft-pbft-consensus/src/bft/sync/mod.rs | 173 ++++++++++++++++-------
 2 files changed, 168 insertions(+), 72 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 6e95d006..bb327e63 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -3,19 +3,13 @@
 //! By default, it is hidden to the user, unless explicitly enabled
 //! with the feature flag `expose_impl`.
 
-use std::fmt::{Debug, Formatter};
+use std::fmt::Debug;
 use std::ops::Drop;
 use std::sync::Arc;
 use std::sync::atomic::AtomicBool;
 use std::time::Instant;
 
-use log::{debug, info, LevelFilter, trace, warn};
-use log4rs::{
-    append::file::FileAppender,
-    config::{Appender, Logger, Root},
-    Config,
-    encode::pattern::PatternEncoder,
-};
+use log::{debug, info, trace, warn};
 
 use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
@@ -24,31 +18,30 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::serialize::Serializable;
-use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::ApplicationData;
-use atlas_core::messages::ClientRqInfo;
 use atlas_core::messages::Protocol;
-use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, ProtocolConsensusDecision, ProtocolMessage, SerProof, SerProofMetadata, View};
+use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, ProtocolConsensusDecision, SerProof, SerProofMetadata, View};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::{ReconfigurableOrderProtocol, ReconfigurationAttemptResult};
 use atlas_core::ordering_protocol::stateful_order_protocol::{DecLog, StatefulOrderProtocol};
 use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol, StatefulOrderingProtocolLog};
-use atlas_core::reconfiguration_protocol::{QuorumJoinCert, ReconfigurationProtocol};
-use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
-use atlas_core::serialize::{LogTransferMessage, NetworkView, OrderingProtocolMessage, ReconfigurationProtocolMessage, ServiceMsg, StateTransferMessage};
-use atlas_core::state_transfer::{Checkpoint};
+use atlas_core::reconfiguration_protocol::ReconfigurationProtocol;
+use atlas_core::request_pre_processing::RequestPreProcessor;
+use atlas_core::serialize::{LogTransferMessage, NetworkView, OrderingProtocolMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
+use atlas_execution::ExecutorHandle;
+use atlas_execution::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_duration;
+
 use crate::bft::config::PBFTConfig;
 use crate::bft::consensus::{Consensus, ConsensusPollStatus, ConsensusStatus, ProposerConsensusGuard};
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, ObserveEventKind, PBFTMessage, ViewChangeMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{CONSENSUS_INSTALL_STATE_TIME_ID, MSG_LOG_INSTALL_TIME_ID};
-use crate::bft::msg_log::{initialize_decided_log};
+use crate::bft::msg_log::initialize_decided_log;
 use crate::bft::msg_log::decided_log::Log;
 use crate::bft::msg_log::decisions::{DecisionLog, Proof};
 use crate::bft::proposer::Proposer;
-use crate::bft::sync::{AbstractSynchronizer, Synchronizer, SynchronizerPollStatus, SynchronizerStatus};
+use crate::bft::sync::{AbstractSynchronizer, Synchronizer, SynchronizerPollStatus, SynchronizerStatus, SyncReconfigurationResult};
 
 pub mod consensus;
 pub mod proposer;
@@ -112,7 +105,7 @@ pub struct PBFTOrderProtocol<D, NT, PL>
     proposer: Arc<Proposer<D, NT>>,
     // The networking layer for a Node in the network (either Client or Replica)
     node: Arc<NT>,
-
+    // The handle to the executor, currently not utilized
     executor: ExecutorHandle<D>,
 }
 
@@ -456,7 +449,9 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
                     SyncPhaseRes::JoinedQuorum(to_execute, node) => {
                         info!("Replica {:?} joined the quorum, with a decision to execute? {}", node, to_execute.is_some());
 
-                        OrderProtocolExecResult::QuorumJoined(to_execute.map(|x| vec![x]), node)
+                        let new_quorum = self.synchronizer.view().quorum_members().clone();
+
+                        OrderProtocolExecResult::QuorumJoined(to_execute.map(|x| vec![x]), node, new_quorum)
                     }
                     SyncPhaseRes::RunCSTProtocol => {
                         OrderProtocolExecResult::RunCst
@@ -842,9 +837,37 @@ impl<D, NT, PL, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, NT,
           NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
           PL: OrderingProtocolLog<PBFTConsensus<D>> {
     fn attempt_quorum_node_join(&mut self, joining_node: NodeId) -> Result<ReconfigurationAttemptResult> {
-        self.switch_phase(ConsensusPhase::SyncPhase);
+        let result = self.synchronizer.start_join_quorum(joining_node, &*self.node, &self.timeouts, &self.message_log);
+
+        return match result {
+            SyncReconfigurationResult::Failed => {
+                warn!("Failed to start quorum view change to integrate node {:?}", joining_node);
+
+                Ok(ReconfigurationAttemptResult::Failed)
+            }
+            SyncReconfigurationResult::OnGoingViewChange => {
 
-        Ok(self.synchronizer.start_join_quorum(joining_node, &*self.node, &self.timeouts, &self.message_log))
+                Ok(ReconfigurationAttemptResult::InProgress)
+            }
+            SyncReconfigurationResult::OnGoingQuorumChange(node_id) if node_id == joining_node  => {
+                warn!("Received join request for node {:?} when it was already ongoing", joining_node);
+
+                Ok(ReconfigurationAttemptResult::CurrentlyReconfiguring(node_id))
+            }
+            SyncReconfigurationResult::OnGoingQuorumChange(node_id) => {
+
+                Ok(ReconfigurationAttemptResult::CurrentlyReconfiguring(node_id))
+            }
+            SyncReconfigurationResult::AlreadyPartOfQuorum => {
+                Ok(ReconfigurationAttemptResult::AlreadyPartOfQuorum)
+            }
+            SyncReconfigurationResult::InProgress => {
+                Ok(ReconfigurationAttemptResult::InProgress)
+            }
+            SyncReconfigurationResult::Completed => {
+                Ok(ReconfigurationAttemptResult::Successful)
+            }
+        };
     }
 
     fn joining_quorum(&mut self) -> Result<ReconfigurationAttemptResult> {
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 038175bb..5cdaab0c 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -6,6 +6,7 @@ use std::{
     time::Duration,
 };
 use std::cell::Cell;
+use std::collections::{BTreeMap, BTreeSet};
 use std::fmt::{Debug, Formatter};
 
 use either::Either;
@@ -364,6 +365,7 @@ pub enum SynchronizerStatus<O> {
     /// The view change protocol just finished running and we
     /// have successfully joined the quorum.
     NewViewJoinedQuorum(Option<ProtocolConsensusDecision<O>>, NodeId),
+
     /// Before we finish the view change protocol, we need
     /// to run the CST protocol.
     RunCst,
@@ -392,6 +394,23 @@ pub enum SynchronizerPollStatus<O> {
     ResumeViewChange,
 }
 
+/// The result of attempting to join a quorum
+#[derive(Clone)]
+pub enum SyncReconfigurationResult {
+    // Something failed when attempting to reconfigure the quorum
+    Failed,
+    // There is already a view change being processed
+    OnGoingViewChange,
+    // There is already a quorum change being processed
+    OnGoingQuorumChange(NodeId),
+    // This node is already a part of the quorum
+    AlreadyPartOfQuorum,
+    // The change is currently in progress
+    InProgress,
+    // We have successfully completed the reconfiguration
+    Completed,
+}
+
 ///A trait describing some of the necessary methods for the synchronizer
 pub trait AbstractSynchronizer<D> where D: ApplicationData + 'static {
     /// Returns information regarding the current view, such as
@@ -420,6 +439,9 @@ pub struct Synchronizer<D: ApplicationData> {
     stopped: RefCell<IntMap<Vec<StoredRequestMessage<D::Request>>>>,
     //Stores currently received requests from other nodes
     currently_adding_node: Cell<Option<NodeId>>,
+    //Stores which nodes are currently being added to the quorum, along with the number of votes
+    //For each of the nodeIDs 
+    currently_adding: RefCell<BTreeMap<NodeId, BTreeSet<NodeId>>>,
     //TODO: This does not require a Mutex I believe since it's only accessed when
     // Processing messages (which is always done in the replica thread)
     collects: Mutex<CollectsType<D>>,
@@ -482,7 +504,6 @@ impl<D> AbstractSynchronizer<D> for Synchronizer<D>
                 // The view change in the log, which means we can't keep executing
                 // Without first processing this sync message
                 self.phase.replace(ProtoPhase::Syncing);
-
             } else {
                 self.tbo.lock().unwrap().install_view(view);
                 return false;
@@ -516,6 +537,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             phase: Cell::new(ProtoPhase::Init),
             stopped: RefCell::new(Default::default()),
             currently_adding_node: Cell::new(None),
+            currently_adding: RefCell::new(Default::default()),
             collects: Mutex::new(Default::default()),
             tbo: Mutex::new(TboQueue::new(view)),
             finalize_state: RefCell::new(None),
@@ -530,6 +552,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             phase: Cell::new(ProtoPhase::Init),
             stopped: RefCell::new(Default::default()),
             currently_adding_node: Cell::new(None),
+            currently_adding: RefCell::new(Default::default()),
             collects: Mutex::new(Default::default()),
             tbo: Mutex::new(TboQueue::new(view)),
             finalize_state: RefCell::new(None),
@@ -552,6 +575,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             tbo: Mutex::new(TboQueue::new(view_info)),
             stopped: RefCell::new(Default::default()),
             currently_adding_node: Cell::new(None),
+            currently_adding: RefCell::new(Default::default()),
             collects: Mutex::new(Default::default()),
             finalize_state: RefCell::new(None),
             entering_quorum: Cell::new(false),
@@ -851,11 +875,6 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
                         return SynchronizerStatus::Running;
                     }
-                    ViewChangeMessageKind::StopQuorumJoin(node_id) if self.currently_adding_node.get().is_some() && self.currently_adding_node.get().unwrap() != *node_id => {
-                        error!("{:?} // Received stop quorum join message with node that does not match the current received  {:?} vs {:?}", node.id(), self.currently_adding_node.get(), node_id);
-
-                        return SynchronizerStatus::Running;
-                    }
                     ViewChangeMessageKind::StopQuorumJoin(node) => (received + 1, *node),
                     ViewChangeMessageKind::StopData(_) => {
                         match &self.accessory {
@@ -889,51 +908,66 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                     }
                 };
 
-                //TODO: Verify that the node join certificate is valid
-
-                if let ProtoPhase::ViewStopping(_received) = self.phase.get() {
-                    return if received > current_view.params().f() {
-                        self.begin_quorum_view_change(None, &**node, timeouts, log);
+                let received_votes = self.currently_adding.borrow_mut().entry(node_id).or_insert_with(BTreeSet::new);
 
-                        SynchronizerStatus::Running
-                    } else {
-                        self.phase.replace(ProtoPhase::ViewStopping(received));
-
-                        SynchronizerStatus::Nil
-                    };
+                if received_votes.insert(header.from()) {
+                    debug!("{:?} // Received stop quorum join message from {:?} with node {:?} ", node.id(), header.from(), node_id);
+                } else {
+                    debug!("{:?} // Received duplicate stop quorum join message from {:?} with node {:?} ", node.id(), header.from(), node_id);
                 }
 
+                // We don't need to actually receive the reconfiguration confirmation to add a node to the quorum, if the quorum is already reached
+                //TODO: Is this the correct procedure?
+                self.phase.replace(ProtoPhase::ViewStopping(received));
+
                 if received >= current_view.params().quorum() {
-                    self.currently_adding_node.replace(Some(node_id));
+                    let mut votes: Vec<_> = self.currently_adding.borrow().iter().map(|(node, voters)| (*node, voters.len())).collect();
 
-                    let next_view = current_view.next_view_with_new_node(self.currently_adding_node.get().unwrap());
+                    votes.sort_by(|(node, votes), (node_2, votes_2)| votes_2.cmp(votes));
 
-                    let previous_view = current_view.clone();
+                    if let Some(vote_count) = votes.first() {
+                        if vote_count.1 >= current_view.params().quorum() {
+                            self.currently_adding_node.replace(Some(node_id));
 
-                    //We have received the necessary amount of stopping requests
-                    //To now that we should move to the next view
+                            let node_to_add = vote_count.0;
 
-                    let next_leader = next_view.leader();
+                            let next_view = current_view.next_view_with_new_node(node_to_add);
 
-                    warn!("{:?} // Stopping quorum reached, moving to next view {:?}. ", node.id(), next_view);
+                            let previous_view = current_view.clone();
 
-                    self.install_next_view(next_view);
+                            //We have received the necessary amount of stopping requests
+                            //To now that we should move to the next view
 
-                    match &self.accessory {
-                        SynchronizerAccessory::Replica(rep) => {
-                            rep.handle_stopping_quorum(self, previous_view, consensus,
-                                                       log, rq_pre_processor, timeouts, &**node)
-                        }
-                        SynchronizerAccessory::Follower(_) => {}
-                    }
+                            let next_leader = next_view.leader();
 
-                    if next_leader == node.id() {
-                        warn!("{:?} // I am the new leader, moving to the stopping data phase.", node.id());
+                            warn!("{:?} // Stopping quorum reached with {} votes for node {:?} moving to next view {:?}. ", node.id(), vote_count.1, node_to_add, next_view);
 
-                        //Move to the stopping data phase as we are the new leader
-                        self.phase.replace(ProtoPhase::StoppingData(0));
-                    } else {
-                        self.phase.replace(ProtoPhase::Syncing);
+                            self.install_next_view(next_view);
+
+                            match &self.accessory {
+                                SynchronizerAccessory::Replica(rep) => {
+                                    rep.handle_stopping_quorum(self, previous_view, consensus,
+                                                               log, rq_pre_processor, timeouts, &**node)
+                                }
+                                SynchronizerAccessory::Follower(_) => {}
+                            }
+
+                            if next_leader == node.id() {
+                                warn!("{:?} // I am the new leader, moving to the stopping data phase.", node.id());
+
+                                //Move to the stopping data phase as we are the new leader
+                                self.phase.replace(ProtoPhase::StoppingData(0));
+                            } else {
+                                self.phase.replace(ProtoPhase::Syncing);
+                            }
+                        } else if received >= current_view.params().n() {
+                            error!("We have received view stopping messages from all nodes in the network and yet we don't have quorum {} votes for any node. {:?}",
+                                   current_view.params().quorum(), votes);
+
+                            todo!("")
+                        } else {
+                            warn!("{:?} // Stopping quorum reached, but not enough votes to add node {:?}. ", node.id(), vote_count.0);
+                        }
                     }
                 } else {
                     self.phase.replace(ProtoPhase::ViewStopping2(received));
@@ -1308,7 +1342,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
     /// Start the quorum join procedure to integrate the given joining node into the current quorum
     /// of the system
-    pub fn start_join_quorum<NT, PL>(&self, joining_node: NodeId, node: &NT, timeouts: &Timeouts, log: &Log<D, PL>) -> ReconfigurationAttemptResult
+    pub fn start_join_quorum<NT, PL>(&self, joining_node: NodeId, node: &NT, timeouts: &Timeouts, log: &Log<D, PL>) -> SyncReconfigurationResult
         where NT: OrderProtocolSendNode<D, PBFT<D>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>> {
         let current_view = self.view();
@@ -1319,7 +1353,31 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             //We are already a part of the quorum, so we don't need to do anything
             info!("{:?} // Attempted to add node {:?} quorum but it is already a part of the quorum", node.id(), joining_node);
 
-            return ReconfigurationAttemptResult::AlreadyPartOfQuorum;
+            return SyncReconfigurationResult::AlreadyPartOfQuorum;
+        }
+
+        match self.phase.get() {
+            ProtoPhase::Init => {
+                // This means this is ready to change views
+            }
+            ProtoPhase::StoppingData(_) | ProtoPhase::SyncingState | ProtoPhase::Syncing => {
+                return if let Some(currently_adding) = self.currently_adding_node.get() {
+                    info!("{:?} // Attempted to add node {:?} quorum but we are currently already adding another node to it {:?}", node.id(), joining_node, currently_adding);
+
+                    SyncReconfigurationResult::OnGoingQuorumChange(currently_adding)
+                } else {
+                    SyncReconfigurationResult::OnGoingViewChange
+                }
+            }
+            ProtoPhase::ViewStopping2(_) | ProtoPhase::Stopping(_) | ProtoPhase::Stopping2(_) => {
+                // Here we still don't know what is the target of the view change.
+                return SyncReconfigurationResult::OnGoingViewChange
+            }
+            _ => {
+                info!("{:?} // Attempted to add node {:?} quorum but we are currently performing a view change", node.id(), joining_node);
+
+                return SyncReconfigurationResult::OnGoingViewChange;
+            }
         }
 
         if joining_node == node.id() {
@@ -1328,7 +1386,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             self.begin_quorum_view_change(Some(joining_node), node, timeouts, log);
         }
 
-        return ReconfigurationAttemptResult::InProgress;
+        return SyncReconfigurationResult::InProgress;
     }
 
     /// Prepare ourselves for the quorum join procedure by stopping the current view and starting a new one
@@ -1344,8 +1402,19 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             info!("{:?} // Attempted to join quorum, but we are already a part of it, can we process sync messages {:?}", node.id(), self.can_process_sync());
 
             return ReconfigurationAttemptResult::AlreadyPartOfQuorum;
+        } else if let Some(next_view) = self.next_view() {
+            if next_view.quorum_members().contains(&node.id()) {
+                //We are already a part of the quorum, so we don't need to do anything
+                info!("{:?} // Attempted to join quorum, but we are already a part of the next view, can we process sync messages {:?}", node.id(), self.can_process_sync());
+
+                return ReconfigurationAttemptResult::AlreadyPartOfQuorum;
+            }
         }
 
+        //TODO: Timeout waiting for the sync/stopping data. This is because
+        // We actually might try to enter while the protocol is running a different view change,
+        // so the view change to integrate us into the quorum might be delayed
+
         // Simulate that we were accepted into the quorum
         let view = current_view.next_view_with_new_node(node.id());
 
@@ -1366,12 +1435,11 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     }
 
     /// Trigger a view change locally
-    pub fn begin_quorum_view_change<NT, PL>(
-        &self,
-        join_cert: Option<NodeId>,
-        node: &NT,
-        timeouts: &Timeouts,
-        _log: &Log<D, PL>, )
+    pub fn begin_quorum_view_change<NT, PL>(&self,
+                                            join_cert: Option<NodeId>,
+                                            node: &NT,
+                                            timeouts: &Timeouts,
+                                            _log: &Log<D, PL>, )
         where NT: OrderProtocolSendNode<D, PBFT<D>>,
               PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
@@ -1379,7 +1447,8 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
         match (self.phase.get(), &join_cert) {
             (ProtoPhase::ViewStopping(i), None) => {
-                self.phase.replace(ProtoPhase::ViewStopping2(i + 1));
+                // We have not received a join certificate message from the node, so we still will
+                self.phase.replace(ProtoPhase::ViewStopping(i + 1));
             }
             (ProtoPhase::ViewStopping(i), _) => {
                 // We have ourselves received a join certificate message from the node, so we will now
@@ -1392,8 +1461,12 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                 return;
             }
             _ => {
+                // If we are called when in the init phase, we should clear the state as a new
+                // View change is going to start.
                 self.stopped.borrow_mut().clear();
                 self.collects.lock().unwrap().clear();
+                self.currently_adding_node.replace(None);
+                self.currently_adding.borrow_mut().clear();
 
                 self.phase.replace(ProtoPhase::ViewStopping2(0));
             }
@@ -1403,8 +1476,6 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             SynchronizerAccessory::Follower(_) => {}
             SynchronizerAccessory::Replica(replica) => {
                 if let Some(join_cert) = join_cert {
-                    self.currently_adding_node.replace(Some(join_cert));
-
                     // We only want to send our STOP message when we have received the notification
                     // From the reconfiguration protocol, even if there are already f+1 STOP messages
                     replica.handle_begin_quorum_view_change(self, timeouts, node, join_cert)
@@ -1470,6 +1541,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                 self.stopped.borrow_mut().clear();
                 self.collects.lock().unwrap().clear();
                 self.currently_adding_node.replace(None);
+                self.currently_adding.borrow_mut().clear();
                 self.entering_quorum.replace(false);
 
                 //Set the new state to be stopping
@@ -1582,6 +1654,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
         if self.currently_adding_node.get().is_some() {
             let node = self.currently_adding_node.replace(None);
+            self.currently_adding.borrow_mut().clear();
 
             SynchronizerStatus::NewViewJoinedQuorum(to_execute, node.unwrap())
         } else {

From 4b6632e77d0e4eccfee31eececb3928bdb4eea28 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 16 Aug 2023 20:12:53 +0100
Subject: [PATCH 37/80] Almost done with new join protocol, still have some
 quirks to smooth out relating to state initialization.

---
 febft-pbft-consensus/src/bft/sync/mod.rs | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 5cdaab0c..e2f20791 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -908,12 +908,16 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                     }
                 };
 
-                let received_votes = self.currently_adding.borrow_mut().entry(node_id).or_insert_with(BTreeSet::new);
+                {
+                    let mut write_guard = self.currently_adding.borrow_mut();
 
-                if received_votes.insert(header.from()) {
-                    debug!("{:?} // Received stop quorum join message from {:?} with node {:?} ", node.id(), header.from(), node_id);
-                } else {
-                    debug!("{:?} // Received duplicate stop quorum join message from {:?} with node {:?} ", node.id(), header.from(), node_id);
+                    let received_votes = write_guard.entry(node_id).or_insert_with(BTreeSet::new);
+
+                    if received_votes.insert(header.from()) {
+                        debug!("{:?} // Received stop quorum join message from {:?} with node {:?} ", node.id(), header.from(), node_id);
+                    } else {
+                        debug!("{:?} // Received duplicate stop quorum join message from {:?} with node {:?} ", node.id(), header.from(), node_id);
+                    }
                 }
 
                 // We don't need to actually receive the reconfiguration confirmation to add a node to the quorum, if the quorum is already reached

From cd5304ad47a178e13226ea24aa72764165979422 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 17 Aug 2023 11:24:35 +0100
Subject: [PATCH 38/80] Fixed issue with proposer sending pre prepare to all
 nodes

---
 febft-pbft-consensus/src/bft/mod.rs          | 2 +-
 febft-pbft-consensus/src/bft/proposer/mod.rs | 6 +++---
 febft-pbft-consensus/src/bft/sync/mod.rs     | 4 +++-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index bb327e63..ee3eb409 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -284,7 +284,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
 
         let sync = Synchronizer::initialize_with_quorum(node_id, view.sequence_number(), quorum.clone(), timeout_dur)?;
 
-        let consensus_guard = ProposerConsensusGuard::new(view.clone(), watermark);
+        let consensus_guard = ProposerConsensusGuard::new(sync.view(), watermark);
 
         let consensus = Consensus::<D, PL>::new_replica(node_id, &sync.view(), executor.clone(),
                                                         SeqNo::ZERO, watermark, consensus_guard.clone(),
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index fc4ca0c6..1b41afeb 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -403,7 +403,9 @@ impl<D, NT> Proposer<D, NT>
             self.consensus_guard.sync_messages_clear();
         }
 
-        info!("{:?} // Proposing new batch with {} request count {:?}", self.node_ref.id(), currently_accumulated.len(), seq);
+        let targets = view.quorum_members().clone();
+
+        info!("{:?} // Proposing new batch with {} request count {:?} to quorum: {:?}", self.node_ref.id(), currently_accumulated.len(), seq, targets);
 
         let message = PBFTMessage::Consensus(ConsensusMessage::new(
             seq,
@@ -411,8 +413,6 @@ impl<D, NT> Proposer<D, NT>
             ConsensusMessageKind::PrePrepare(currently_accumulated),
         ));
 
-        let targets = view.quorum_members().clone();
-
         self.node_ref.broadcast_signed(message, targets.into_iter());
 
         metric_increment(PROPOSER_BATCHES_MADE_ID, Some(1));
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index e2f20791..7d3a4ee9 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -567,7 +567,9 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
         let f = (n - 1) / 3;
 
-        let view_info = ViewInfo::new(seq_no, n, f)?;
+        let view_info = ViewInfo::from_quorum(seq_no, quorum_members)?;
+
+        info!("Initializing synchronizer with view {:?}", view_info);
 
         Ok(Arc::new(Self {
             node_id,

From d8010b595012cbe442408da3af7b311bf0958455 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 17 Aug 2023 12:51:13 +0100
Subject: [PATCH 39/80] Fixed issue when resuming view change which was not
 returning a node join so the reconfiguration never received confirmation

---
 febft-pbft-consensus/src/bft/mod.rs      | 20 +++++++++++++++-----
 febft-pbft-consensus/src/bft/sync/mod.rs | 10 +++++-----
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index ee3eb409..6de7c620 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -345,7 +345,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
             SynchronizerPollStatus::ResumeViewChange => {
                 debug!("{:?} // Resuming view change", self.node.id());
 
-                self.synchronizer.resume_view_change(
+                let sync_status = self.synchronizer.resume_view_change(
                     &mut self.message_log,
                     &self.timeouts,
                     &mut self.consensus,
@@ -354,6 +354,16 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
 
                 self.switch_phase(ConsensusPhase::NormalPhase);
 
+                if let Some(sync_status) = sync_status {
+                    match sync_status {
+                        SynchronizerStatus::NewViewJoinedQuorum(decisions, node) => {
+                            let quorum_members = self.synchronizer.view().quorum_members().clone();
+                            return OrderProtocolPoll::QuorumJoined(decisions.map(|dec| vec![dec]), node, quorum_members);
+                        }
+                        _ => {}
+                    }
+                }
+
                 OrderProtocolPoll::RePoll
             }
         }
@@ -397,7 +407,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
                             warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, Protocol Finished");
                             OrderProtocolPoll::RePoll
                         }
-                    }
+                    };
                 } else {
                     // The synchronizer should never return anything other than a view
                     // change message
@@ -611,6 +621,8 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
 
                 //After we update the state, we go back to the sync phase (this phase) so we can check if we are missing
                 //Anything or to finalize and go back to the normal phase
+                info!("Running CST protocol as requested by the synchronizer");
+
                 self.switch_phase(ConsensusPhase::SyncPhase);
 
                 SyncPhaseRes::RunCSTProtocol
@@ -846,16 +858,14 @@ impl<D, NT, PL, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, NT,
                 Ok(ReconfigurationAttemptResult::Failed)
             }
             SyncReconfigurationResult::OnGoingViewChange => {
-
                 Ok(ReconfigurationAttemptResult::InProgress)
             }
-            SyncReconfigurationResult::OnGoingQuorumChange(node_id) if node_id == joining_node  => {
+            SyncReconfigurationResult::OnGoingQuorumChange(node_id) if node_id == joining_node => {
                 warn!("Received join request for node {:?} when it was already ongoing", joining_node);
 
                 Ok(ReconfigurationAttemptResult::CurrentlyReconfiguring(node_id))
             }
             SyncReconfigurationResult::OnGoingQuorumChange(node_id) => {
-
                 Ok(ReconfigurationAttemptResult::CurrentlyReconfiguring(node_id))
             }
             SyncReconfigurationResult::AlreadyPartOfQuorum => {
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 7d3a4ee9..bd7b8340 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -1323,16 +1323,17 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         timeouts: &Timeouts,
         consensus: &mut Consensus<D, PL>,
         node: &Arc<NT>,
-    ) -> Option<()>
+    ) -> Option<SynchronizerStatus<D::Request>>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
               PL: OrderingProtocolLog<PBFTConsensus<D>>
     {
+
         let state = self.finalize_state.borrow_mut().take()?;
 
         //This is kept alive until it is out of the scope
         let _lock_guard = self.collects.lock().unwrap();
 
-        finalize_view_change!(
+        Some(finalize_view_change!(
             self,
             state,
             None,
@@ -1341,9 +1342,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             timeouts,
             consensus,
             node,
-        );
-
-        Some(())
+        ))
     }
 
     /// Start the quorum join procedure to integrate the given joining node into the current quorum
@@ -1660,6 +1659,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
         if self.currently_adding_node.get().is_some() {
             let node = self.currently_adding_node.replace(None);
+
             self.currently_adding.borrow_mut().clear();
 
             SynchronizerStatus::NewViewJoinedQuorum(to_execute, node.unwrap())

From c59f6f2026c66deb5c6e8776909118379febd062 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 17 Aug 2023 14:55:13 +0100
Subject: [PATCH 40/80] Fixed some debug messages, tested some more things.

---
 febft-pbft-consensus/src/bft/consensus/mod.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 4ded90b8..7d2d48d5 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -276,7 +276,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             Either::Right(_) => {}
             Either::Left(_) => {
                 // The message pertains to older views
-                warn!("{:?} // Ignoring consensus message {:?} received from {:?} as we are already in view {:?}",
+                debug!("{:?} // Ignoring consensus message {:?} received from {:?} as we are already in view {:?}",
                     self.node_id, message, header.from(), self.curr_view.sequence_number());
 
                 return;
@@ -288,7 +288,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             Either::Left(_) => {
                 // The message pertains to older consensus instances
 
-                warn!("{:?} // Ignoring consensus message {:?} received from {:?} as we are already in seq no {:?}",
+                debug!("{:?} // Ignoring consensus message {:?} received from {:?} as we are already in seq no {:?}",
                     self.node_id, message, header.from(), self.seq_no);
 
                 return;
@@ -382,7 +382,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             Either::Right(_) => {}
             Either::Left(_) => {
                 // The message pertains to older views
-                warn!("{:?} // Ignoring consensus message {:?} received from {:?} as we are already in view {:?}",
+                debug!("{:?} // Ignoring consensus message {:?} received from {:?} as we are already in view {:?}",
                     self.node_id, message, header.from(), self.curr_view.sequence_number());
 
                 return Ok(ConsensusStatus::Deciding);
@@ -392,7 +392,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
         let i = match message_seq.index(self.seq_no) {
             Either::Right(i) => i,
             Either::Left(_) => {
-                warn!("Message {:?} from {:?} is behind our current sequence no {:?}. Ignoring", message, header.from(), self.seq_no, );
+                debug!("Message {:?} from {:?} is behind our current sequence no {:?}. Ignoring", message, header.from(), self.seq_no, );
 
                 return Ok(ConsensusStatus::Deciding);
             }

From 402b48de26f034be03bea4aa0e19bb272836a76a Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 17 Aug 2023 16:00:31 +0100
Subject: [PATCH 41/80] Fix state transferring and made to count votes by
 digest.

---
 febft-state-transfer/src/lib.rs | 64 ++++++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 21 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 82cab562..b658b9f2 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -5,7 +5,7 @@ use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-use log::{debug, error, info};
+use log::{debug, error, info, warn};
 #[cfg(feature = "serialize_serde")]
 use serde::{Deserialize, Serialize};
 use atlas_common::channel::ChannelSyncTx;
@@ -94,7 +94,7 @@ impl<S> Debug for ProtoPhase<S> {
 /// and because decision log also got a lot easier to clone, but we still have
 /// to be very careful thanks to the requests vector, which can be VERY large
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct RecoveryState<S> {
     pub checkpoint: Arc<ReadOnly<Checkpoint<S>>>,
 }
@@ -115,11 +115,13 @@ impl<S> RecoveryState<S> {
     }
 }
 
+#[derive(Debug)]
 struct ReceivedState<S> {
     count: usize,
     state: RecoveryState<S>,
 }
 
+#[derive(Debug)]
 struct ReceivedStateCid {
     cid: SeqNo,
     count: usize,
@@ -135,8 +137,6 @@ pub struct CollabStateTransfer<S, NT, PL>
     where S: MonolithicState + 'static {
     curr_seq: SeqNo,
     current_checkpoint_state: CheckpointState<S>,
-    largest_cid: SeqNo,
-    latest_cid_count: usize,
     base_timeout: Duration,
     curr_timeout: Duration,
     timeouts: Timeouts,
@@ -440,8 +440,6 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
             received_states: collections::hash_map(),
             received_state_ids: collections::hash_map(),
             phase: ProtoPhase::Init,
-            largest_cid: SeqNo::ZERO,
-            latest_cid_count: 0,
             curr_seq: SeqNo::ZERO,
             persistent_log,
             install_channel,
@@ -618,6 +616,9 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                 match message.kind() {
                     CstMessageKind::ReplyStateCid(state_cid) => {
                         if let Some((cid, digest)) = state_cid {
+                            debug!("{:?} // Received state cid {:?} with digest {:?} from {:?} with seq {:?}",
+                            self.node.id(), state_cid, digest, header.from(), cid);
+
                             let received_state_cid = self.received_state_ids.entry(digest.clone()).or_insert_with(|| {
                                 ReceivedStateCid {
                                     cid: *cid,
@@ -637,6 +638,10 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
 
                                 received_state_cid.count += 1;
                             }
+                        } else {
+
+
+                            debug!("{:?} // Received blank state cid from node {:?}", self.node.id(), header.from());
                         }
                     }
                     CstMessageKind::RequestStateCid => {
@@ -659,29 +664,48 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                 // TODO: check for more than one reply from the same node
                 let i = i + 1;
 
-                debug!("{:?} // Quorum count {}, i: {}, cst_seq {:?}. Current Latest Cid: {:?}. Current Latest Cid Count: {}",
+                debug!("{:?} // Quorum count {}, i: {}, cst_seq {:?}. Current Latest Cid: {:?}",
                         self.node.id(), view.quorum(), i,
-                        self.curr_seq, self.largest_cid, self.latest_cid_count);
+                        self.curr_seq, self.received_state_ids);
 
-                if i == view.quorum() {
+                if i >= view.quorum() {
                     self.phase = ProtoPhase::Init;
 
                     // reset timeout, since req was successful
                     self.curr_timeout = self.base_timeout;
 
-                    info!("{:?} // Identified the latest state seq no as {:?} with {} votes.",
-                            self.node.id(),
-                            self.largest_cid, self.latest_cid_count);
+                    let mut received_state_ids: Vec<_> = self.received_state_ids.iter().map(|(digest, cid)| {
+                        (digest, cid.cid, cid.count)
+                    }).collect();
+
+                    received_state_ids.sort_by(|(_, _, count), (_, _, count2)| {
+                        count.cmp(count2).reverse()
+                    });
+
+                    if let Some((digest, seq, count)) = received_state_ids.first() {
+                        if *count >= view.quorum() {
+                            info!("{:?} // Received quorum of states for CST Seq {:?} with digest {:?} and seq {:?}",
+                                self.node.id(), self.curr_seq, digest, seq);
+
+                            return CstStatus::SeqNo(*seq);
+                        } else {
+                            warn!("Received quorum state messages but we still don't have a quorum of states? Faulty replica? {:?}", self.received_state_ids)
+                        }
+                    } else {
+                        // If we are completely blank, then no replicas have state, so we can initialize
+
+                        warn!("We have received a quorum of blank messages, which means we are probably at the start");
+                        return CstStatus::SeqNo(SeqNo::ZERO);
+                    }
 
                     // we don't need the latest cid to be available in at least
                     // f+1 replicas since the replica has the proof that the system
                     // has decided
-                    CstStatus::SeqNo(self.largest_cid)
-                } else {
-                    self.phase = ProtoPhase::ReceivingCid(i);
-
-                    CstStatus::Running
                 }
+
+                self.phase = ProtoPhase::ReceivingCid(i);
+
+                CstStatus::Running
             }
             ProtoPhase::ReceivingState(i) => {
                 let (header, mut message) = getmessage!(progress, CstStatus::RequestState);
@@ -886,10 +910,8 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
         view: V,
     ) where V: NetworkView
     {
-
-        // reset state of latest seq no. request
-        self.largest_cid = SeqNo::ZERO;
-        self.latest_cid_count = 0;
+        // Reset the map of received state ids
+        self.received_state_ids.clear();
 
         self.next_seq();
 

From bfcb84f02e47b01b1e2c583ec48514e69e40341e Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 17 Aug 2023 17:59:35 +0100
Subject: [PATCH 42/80] Added warnings to see when state and log transfers get
 re trigerred when running.

---
 febft-state-transfer/src/lib.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index b658b9f2..3e6661a0 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -801,6 +801,9 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                     Some(ReceivedState { count, state }) if count > f => {
                         self.phase = ProtoPhase::Init;
 
+                        info!("{:?} // Received quorum of states for CST Seq {:?} with digest {:?}, returning the state to the replica",
+                            self.node.id(), self.curr_seq, digest);
+
                         CstStatus::State(state)
                     }
                     _ => {

From 9a6dcb163f841e659808b45bd2d5af46d30fd558 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Fri, 18 Aug 2023 18:45:47 +0100
Subject: [PATCH 43/80] Start working on recursive message signature
 verification.

---
 .../src/bft/message/serialize/mod.rs          | 79 ++++++++++++++++++-
 febft-pbft-consensus/src/bft/sync/mod.rs      |  5 ++
 2 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index 0ecc4668..c19a1847 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -14,13 +14,15 @@ use ::serde::{Deserialize, Serialize};
 use bytes::Bytes;
 
 use atlas_common::error::*;
+use atlas_communication::message::Header;
 use atlas_communication::serialize::Serializable;
+use atlas_core::messages::SystemMessage;
 use atlas_core::persistent_log::PersistableOrderProtocol;
 use atlas_core::reconfiguration_protocol::QuorumJoinCert;
-use atlas_core::serialize::{OrderingProtocolMessage, ReconfigurationProtocolMessage, StatefulOrderProtocolMessage};
+use atlas_core::serialize::{OrderingProtocolMessage, ReconfigurationProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage};
 use atlas_execution::serialize::ApplicationData;
 
-use crate::bft::message::{ConsensusMessage, PBFTMessage};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata};
 use crate::bft::sync::view::ViewInfo;
 
@@ -64,6 +66,65 @@ pub fn deserialize_consensus<R, D>(r: R) -> Result<ConsensusMessage<D::Request>>
 /// The serializable type, to be used to appease the compiler and it's requirements
 pub struct PBFTConsensus<D: ApplicationData>(PhantomData<(D)>);
 
+impl<D> PBFTConsensus<D> where D: ApplicationData {
+
+    /// Verify the consensus internal message structure
+    fn verify_consensus_message(header: &Header, msg: &ConsensusMessage<D::Request>) -> Result<bool> {
+        match msg.kind() {
+            ConsensusMessageKind::PrePrepare(requests) => {
+                requests.iter().map(|request| {
+                    let header = request.header();
+                    let client_request = request.message();
+
+                    Ok(false)
+                }).reduce(|a, b| a.and(b))
+                    .unwrap_or(Ok(true))
+            }
+            ConsensusMessageKind::Prepare(prepare) => {
+                Ok(true)
+            }
+            ConsensusMessageKind::Commit(commit) => {
+                Ok(true)
+            }
+        }
+    }
+
+    /// Verify view change message internal structure
+    fn verify_view_change_message(header: &Header, msg: &ViewChangeMessage<D::Request>) -> Result<bool> {
+        match msg.kind() {
+            ViewChangeMessageKind::Stop(timed_out_rqs) => {
+                timed_out_rqs.iter().map(|request| {
+                    let header = request.header();
+                    let client_request = request.message();
+
+                    Ok(false)
+                }).reduce(|a, b| a.and(b)).unwrap_or(Ok(true))
+            }
+            ViewChangeMessageKind::StopQuorumJoin(joining_node) => {
+                Ok(true)
+            }
+            ViewChangeMessageKind::StopData(data) => {
+                if let Some(proof) = &data.last_proof {
+                    proof.pre_prepares().iter().map(|pre_prepare| {
+                        let header = pre_prepare.header();
+                        let consensus_message = pre_prepare.message();
+
+                        Self::verify_consensus_message(header, consensus_message)
+                    }).reduce(|a, b| a.and(b)).unwrap_or(Ok(true))
+                }
+            }
+            ViewChangeMessageKind::Sync(sync_message) => {
+                let message = sync_message.message();
+
+                let header = message.header();
+                let message = message.consensus();
+
+                Self::verify_consensus_message(header, message)
+            }
+        }
+    }
+}
+
 impl<D> OrderingProtocolMessage for PBFTConsensus<D>
     where D: ApplicationData, {
     type ViewInfo = ViewInfo;
@@ -72,6 +133,20 @@ impl<D> OrderingProtocolMessage for PBFTConsensus<D>
     type Proof = Proof<D::Request>;
     type ProofMetadata = ProofMetadata;
 
+    fn verify_protocol_message(header: &Header, msg: &Self::ProtocolMessage) -> Result<bool> {
+        match msg {
+            PBFTMessage::Consensus(consensus_msg) => {
+                Self::verify_consensus_message(header, consensus_msg)
+            }
+            PBFTMessage::ViewChange(view_change) => {
+                Self::verify_view_change_message(header, view_change)
+            }
+            PBFTMessage::ObserverMessage(_) => {
+                Ok(true)
+            }
+        }
+    }
+
     #[cfg(feature = "serialize_capnp")]
     fn serialize_capnp(builder: atlas_capnp::consensus_messages_capnp::protocol_message::Builder, msg: &Self::ProtocolMessage) -> Result<()> {
         capnp::serialize_message::<D>(builder, msg)
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index bd7b8340..e978f765 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -123,6 +123,11 @@ pub struct LeaderCollects<O> {
 }
 
 impl<O> LeaderCollects<O> {
+
+    pub fn message(&self) -> &FwdConsensusMessage<O> {
+        &self.proposed
+    }
+
     /// Gives up ownership of the inner values of this `LeaderCollects`.
     pub fn into_inner(
         self,

From 3cd10973b1e3ac336ab6366dcf88533db91b42b7 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Sat, 19 Aug 2023 16:47:48 +0100
Subject: [PATCH 44/80] Fixed problem with connection index finding, causing
 infinite loops in the network layer.

---
 .../src/bft/message/serialize/mod.rs          | 51 ++++++++++++-------
 .../src/message/serialize/mod.rs              | 16 +++++-
 2 files changed, 49 insertions(+), 18 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index c19a1847..6de3c990 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -8,6 +8,7 @@
 
 use std::io::{Read, Write};
 use std::marker::PhantomData;
+use std::sync::Arc;
 
 #[cfg(feature = "serialize_serde")]
 use ::serde::{Deserialize, Serialize};
@@ -15,11 +16,13 @@ use bytes::Bytes;
 
 use atlas_common::error::*;
 use atlas_communication::message::Header;
+use atlas_communication::message_signing::NetworkMessageSignatureVerifier;
+use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_communication::serialize::Serializable;
 use atlas_core::messages::SystemMessage;
 use atlas_core::persistent_log::PersistableOrderProtocol;
 use atlas_core::reconfiguration_protocol::QuorumJoinCert;
-use atlas_core::serialize::{OrderingProtocolMessage, ReconfigurationProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage};
+use atlas_core::serialize::{InternallyVerifiable, OrderingProtocolMessage, ReconfigurationProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage};
 use atlas_execution::serialize::ApplicationData;
 
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
@@ -67,15 +70,19 @@ pub fn deserialize_consensus<R, D>(r: R) -> Result<ConsensusMessage<D::Request>>
 pub struct PBFTConsensus<D: ApplicationData>(PhantomData<(D)>);
 
 impl<D> PBFTConsensus<D> where D: ApplicationData {
-
     /// Verify the consensus internal message structure
-    fn verify_consensus_message(header: &Header, msg: &ConsensusMessage<D::Request>) -> Result<bool> {
+    fn verify_consensus_message<S, SV, NI>(network_info: &Arc<NI>, header: &Header, msg: &ConsensusMessage<D::Request>) -> Result<bool>
+        where S: Serializable,
+              NI: NetworkInformationProvider,
+              SV: NetworkMessageSignatureVerifier<S, NI>, {
         match msg.kind() {
             ConsensusMessageKind::PrePrepare(requests) => {
                 requests.iter().map(|request| {
                     let header = request.header();
                     let client_request = request.message();
 
+                    //TODO: Verify client request signature
+
                     Ok(false)
                 }).reduce(|a, b| a.and(b))
                     .unwrap_or(Ok(true))
@@ -90,7 +97,10 @@ impl<D> PBFTConsensus<D> where D: ApplicationData {
     }
 
     /// Verify view change message internal structure
-    fn verify_view_change_message(header: &Header, msg: &ViewChangeMessage<D::Request>) -> Result<bool> {
+    fn verify_view_change_message<S, SV, NI>(network_info: &Arc<NI>, header: &Header, msg: &ViewChangeMessage<D::Request>) -> Result<bool>
+        where S: Serializable,
+              NI: NetworkInformationProvider,
+              SV: NetworkMessageSignatureVerifier<S, NI>, {
         match msg.kind() {
             ViewChangeMessageKind::Stop(timed_out_rqs) => {
                 timed_out_rqs.iter().map(|request| {
@@ -109,8 +119,10 @@ impl<D> PBFTConsensus<D> where D: ApplicationData {
                         let header = pre_prepare.header();
                         let consensus_message = pre_prepare.message();
 
-                        Self::verify_consensus_message(header, consensus_message)
+                        Self::verify_consensus_message::<S, SV, NI>(network_info, header, consensus_message)
                     }).reduce(|a, b| a.and(b)).unwrap_or(Ok(true))
+                } else {
+                    Ok(true)
                 }
             }
             ViewChangeMessageKind::Sync(sync_message) => {
@@ -119,33 +131,38 @@ impl<D> PBFTConsensus<D> where D: ApplicationData {
                 let header = message.header();
                 let message = message.consensus();
 
-                Self::verify_consensus_message(header, message)
+                Self::verify_consensus_message::<S, SV, NI>(network_info, header, message)
             }
         }
     }
 }
 
-impl<D> OrderingProtocolMessage for PBFTConsensus<D>
-    where D: ApplicationData, {
-    type ViewInfo = ViewInfo;
-    type ProtocolMessage = PBFTMessage<D::Request>;
-    type LoggableMessage = ConsensusMessage<D::Request>;
-    type Proof = Proof<D::Request>;
-    type ProofMetadata = ProofMetadata;
-
-    fn verify_protocol_message(header: &Header, msg: &Self::ProtocolMessage) -> Result<bool> {
+impl<D> InternallyVerifiable<PBFTMessage<D::Request>> for PBFTConsensus<D> where D: ApplicationData {
+    fn verify_internal_message<S, SV, NI>(network_info: &Arc<NI>, header: &Header, msg: &PBFTMessage<D::Request>) -> Result<bool>
+        where S: Serializable,
+              SV: NetworkMessageSignatureVerifier<S, NI>,
+              NI: NetworkInformationProvider {
         match msg {
             PBFTMessage::Consensus(consensus_msg) => {
-                Self::verify_consensus_message(header, consensus_msg)
+                Self::verify_consensus_message::<S, SV, NI>(network_info, header, consensus_msg)
             }
             PBFTMessage::ViewChange(view_change) => {
-                Self::verify_view_change_message(header, view_change)
+                Self::verify_view_change_message::<S, SV, NI>(network_info, header, view_change)
             }
             PBFTMessage::ObserverMessage(_) => {
                 Ok(true)
             }
         }
     }
+}
+
+impl<D> OrderingProtocolMessage for PBFTConsensus<D>
+    where D: ApplicationData, {
+    type ViewInfo = ViewInfo;
+    type ProtocolMessage = PBFTMessage<D::Request>;
+    type LoggableMessage = ConsensusMessage<D::Request>;
+    type Proof = Proof<D::Request>;
+    type ProofMetadata = ProofMetadata;
 
     #[cfg(feature = "serialize_capnp")]
     fn serialize_capnp(builder: atlas_capnp::consensus_messages_capnp::protocol_message::Builder, msg: &Self::ProtocolMessage) -> Result<()> {
diff --git a/febft-state-transfer/src/message/serialize/mod.rs b/febft-state-transfer/src/message/serialize/mod.rs
index 516d5917..467d64da 100644
--- a/febft-state-transfer/src/message/serialize/mod.rs
+++ b/febft-state-transfer/src/message/serialize/mod.rs
@@ -2,9 +2,14 @@
 mod capnp;
 
 use std::marker::PhantomData;
+use std::sync::Arc;
 use std::time::Instant;
+use atlas_communication::message::Header;
+use atlas_communication::message_signing::NetworkMessageSignatureVerifier;
+use atlas_communication::reconfiguration_node::NetworkInformationProvider;
+use atlas_communication::serialize::Serializable;
 use atlas_execution::serialize::ApplicationData;
-use atlas_core::serialize::{OrderingProtocolMessage, StatefulOrderProtocolMessage, StateTransferMessage};
+use atlas_core::serialize::{InternallyVerifiable, OrderingProtocolMessage, StatefulOrderProtocolMessage, StateTransferMessage};
 use atlas_core::state_transfer::{StateTransferProtocol};
 use atlas_execution::state::monolithic_state::MonolithicState;
 use crate::CollabStateTransfer;
@@ -12,6 +17,15 @@ use crate::message::CstMessage;
 
 pub struct CSTMsg<S: MonolithicState>(PhantomData<(S)>);
 
+impl<S: MonolithicState> InternallyVerifiable<CstMessage<S>> for CSTMsg<S> {
+    fn verify_internal_message<M, SV, NI>(network_info: &Arc<NI>, header: &Header, msg: &CstMessage<S>) -> atlas_common::error::Result<bool>
+        where M: Serializable,
+              SV: NetworkMessageSignatureVerifier<M, NI>,
+              NI: NetworkInformationProvider {
+        Ok(true)
+    }
+}
+
 impl<S: MonolithicState> StateTransferMessage for CSTMsg<S> {
 
     type StateTransferMessage = CstMessage<S>;

From 3c449bd0582de7f04540d8e06fea7c33088b2f96 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Fri, 25 Aug 2023 18:50:20 +0100
Subject: [PATCH 45/80] Fixed compilation for signature signing.

---
 febft-pbft-consensus/src/bft/config/mod.rs    |  6 ++--
 .../src/bft/consensus/accessory/mod.rs        |  1 -
 .../bft/consensus/accessory/replica/mod.rs    |  9 ++----
 .../src/bft/consensus/decision/mod.rs         |  4 +--
 febft-pbft-consensus/src/bft/consensus/mod.rs |  3 +-
 .../src/bft/message/serialize/mod.rs          | 28 ++++------------
 febft-pbft-consensus/src/bft/mod.rs           |  3 +-
 .../src/bft/msg_log/decided_log/mod.rs        |  9 ++----
 .../src/bft/msg_log/decisions/mod.rs          |  2 +-
 febft-pbft-consensus/src/bft/msg_log/mod.rs   | 12 ++-----
 febft-pbft-consensus/src/bft/proposer/mod.rs  |  4 +--
 febft-pbft-consensus/src/bft/sync/mod.rs      |  6 ++--
 .../src/bft/sync/replica_sync/mod.rs          | 22 ++++++-------
 febft-pbft-consensus/src/bft/sync/view/mod.rs |  2 +-
 febft-state-transfer/src/lib.rs               | 17 +++++-----
 .../src/message/serialize/capnp/mod.rs        |  1 -
 .../src/message/serialize/mod.rs              | 32 +++++++------------
 17 files changed, 52 insertions(+), 109 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/config/mod.rs b/febft-pbft-consensus/src/bft/config/mod.rs
index f6f4c007..3e67cf37 100644
--- a/febft-pbft-consensus/src/bft/config/mod.rs
+++ b/febft-pbft-consensus/src/bft/config/mod.rs
@@ -1,16 +1,14 @@
-use std::marker::PhantomData;
 use std::time::Duration;
 
 use atlas_common::node_id::NodeId;
 use atlas_core::followers::FollowerHandle;
-use atlas_core::serialize::{OrderingProtocolMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_execution::serialize::ApplicationData;
 
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::sync::view::ViewInfo;
 
 pub struct PBFTConfig<D>
-    where D: ApplicationData{
+    where D: ApplicationData {
     pub node_id: NodeId,
     // pub observer_handle: ObserverHandle,
     pub follower_handle: Option<FollowerHandle<PBFTConsensus<D>>>,
@@ -20,7 +18,7 @@ pub struct PBFTConfig<D>
     pub watermark: u32,
 }
 
-impl<D: ApplicationData + 'static,> PBFTConfig<D> {
+impl<D: ApplicationData + 'static, > PBFTConfig<D> {
     pub fn new(node_id: NodeId,
                follower_handle: Option<FollowerHandle<PBFTConsensus<D>>>,
                view: ViewInfo, timeout_dur: Duration,
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
index dc663849..5514ad7d 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
@@ -1,7 +1,6 @@
 use std::sync::Arc;
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_execution::serialize::ApplicationData;
 
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index b998f1fa..1036b9c1 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -1,5 +1,4 @@
 use std::collections::BTreeMap;
-use std::ops::Deref;
 use std::sync::{Arc, Mutex};
 
 use chrono::Utc;
@@ -8,20 +7,16 @@ use log::debug;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::threadpool;
-use atlas_communication::message::{NetworkMessageKind, SerializedMessage, StoredMessage, StoredSerializedProtocolMessage, WireMessage};
-use atlas_communication::protocol_node::ProtocolNetworkNode;
+use atlas_communication::message::{SerializedMessage, StoredMessage, StoredSerializedProtocolMessage, WireMessage};
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
-use atlas_core::messages::SystemMessage;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, ServiceMsg, StateTransferMessage};
 use atlas_execution::serialize::ApplicationData;
 
+use crate::bft::{PBFT, SysMsg};
 use crate::bft::consensus::accessory::AccessoryConsensus;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::msg_log::deciding_log::DecidingLog;
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
-use crate::bft::{PBFT, SysMsg};
-use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::sync::view::ViewInfo;
 
 pub struct ReplicaAccessory<D>
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 957f4ed4..1f9dbbfe 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -11,18 +11,16 @@ use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
-use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog};
-use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
 use atlas_execution::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_duration;
 
 use crate::bft::consensus::accessory::{AccessoryConsensus, ConsensusDecisionAccessory};
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{ConsensusMetrics, PRE_PREPARE_ANALYSIS_ID};
 use crate::bft::msg_log::decided_log::Log;
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 7d2d48d5..4f934d72 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -12,11 +12,10 @@ use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_advance_message_queue_return, tbo_queue_message};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
-use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage, SystemMessage};
+use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
 use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index 6de3c990..8468df4b 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -19,10 +19,9 @@ use atlas_communication::message::Header;
 use atlas_communication::message_signing::NetworkMessageSignatureVerifier;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_communication::serialize::Serializable;
-use atlas_core::messages::SystemMessage;
+use atlas_core::ordering_protocol::networking::serialize::{OrderingProtocolMessage, StatefulOrderProtocolMessage};
+use atlas_core::ordering_protocol::networking::signature_ver::OrderProtocolSignatureVerificationHelper;
 use atlas_core::persistent_log::PersistableOrderProtocol;
-use atlas_core::reconfiguration_protocol::QuorumJoinCert;
-use atlas_core::serialize::{InternallyVerifiable, OrderingProtocolMessage, ReconfigurationProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage};
 use atlas_execution::serialize::ApplicationData;
 
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
@@ -137,25 +136,6 @@ impl<D> PBFTConsensus<D> where D: ApplicationData {
     }
 }
 
-impl<D> InternallyVerifiable<PBFTMessage<D::Request>> for PBFTConsensus<D> where D: ApplicationData {
-    fn verify_internal_message<S, SV, NI>(network_info: &Arc<NI>, header: &Header, msg: &PBFTMessage<D::Request>) -> Result<bool>
-        where S: Serializable,
-              SV: NetworkMessageSignatureVerifier<S, NI>,
-              NI: NetworkInformationProvider {
-        match msg {
-            PBFTMessage::Consensus(consensus_msg) => {
-                Self::verify_consensus_message::<S, SV, NI>(network_info, header, consensus_msg)
-            }
-            PBFTMessage::ViewChange(view_change) => {
-                Self::verify_view_change_message::<S, SV, NI>(network_info, header, view_change)
-            }
-            PBFTMessage::ObserverMessage(_) => {
-                Ok(true)
-            }
-        }
-    }
-}
-
 impl<D> OrderingProtocolMessage for PBFTConsensus<D>
     where D: ApplicationData, {
     type ViewInfo = ViewInfo;
@@ -164,6 +144,10 @@ impl<D> OrderingProtocolMessage for PBFTConsensus<D>
     type Proof = Proof<D::Request>;
     type ProofMetadata = ProofMetadata;
 
+    fn verify_order_protocol_message<NI, OPVH, D2>(network_info: &NI, header: &Header, message: Self::ProtocolMessage) -> Result<(bool, Self::ProtocolMessage)> where NI: NetworkInformationProvider, OPVH: OrderProtocolSignatureVerificationHelper<D2, Self, NI>, D2: ApplicationData, Self: Sized {
+        todo!()
+    }
+
     #[cfg(feature = "serialize_capnp")]
     fn serialize_capnp(builder: atlas_capnp::consensus_messages_capnp::protocol_message::Builder, msg: &Self::ProtocolMessage) -> Result<()> {
         capnp::serialize_message::<D>(builder, msg)
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 6de7c620..f1ec36f2 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -21,12 +21,13 @@ use atlas_communication::serialize::Serializable;
 use atlas_core::messages::Protocol;
 use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, ProtocolConsensusDecision, SerProof, SerProofMetadata, View};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
+use atlas_core::ordering_protocol::networking::serialize::{NetworkView, OrderingProtocolMessage};
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::{ReconfigurableOrderProtocol, ReconfigurationAttemptResult};
 use atlas_core::ordering_protocol::stateful_order_protocol::{DecLog, StatefulOrderProtocol};
 use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol, StatefulOrderingProtocolLog};
 use atlas_core::reconfiguration_protocol::ReconfigurationProtocol;
 use atlas_core::request_pre_processing::RequestPreProcessor;
-use atlas_core::serialize::{LogTransferMessage, NetworkView, OrderingProtocolMessage, ReconfigurationProtocolMessage, StateTransferMessage};
+use atlas_core::serialize::ReconfigurationProtocolMessage;
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_execution::ExecutorHandle;
 use atlas_execution::serialize::ApplicationData;
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index 939fffc3..65c58f30 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -1,23 +1,18 @@
-use std::sync::Arc;
-
 use log::error;
 
 use atlas_common::error::*;
-use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_communication::message::StoredMessage;
 use atlas_core::ordering_protocol::{DecisionInformation, ProtocolConsensusDecision};
 use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_core::serialize::{ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_execution::app::UpdateBatch;
 use atlas_execution::serialize::ApplicationData;
 
-use crate::bft::message::{ConsensusMessageKind, PBFTMessage};
+use crate::bft::message::ConsensusMessageKind;
 use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::msg_log::operation_key;
 use crate::bft::msg_log::deciding_log::CompletedBatch;
 use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata, StoredConsensusMessage};
+use crate::bft::msg_log::operation_key;
 use crate::bft::sync::view::ViewInfo;
 
 /// The log of decisions that have already been processed by the consensus
diff --git a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
index 2a8af4d6..b9f04fb9 100644
--- a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
@@ -10,7 +10,7 @@ use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::StoredMessage;
-use atlas_core::serialize::{OrderProtocolLog, OrderProtocolProof};
+use atlas_core::ordering_protocol::networking::serialize::{OrderProtocolLog, OrderProtocolProof};
 
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::msg_log::deciding_log::CompletedBatch;
diff --git a/febft-pbft-consensus/src/bft/msg_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/mod.rs
index 8f1fd3cd..529849b1 100644
--- a/febft-pbft-consensus/src/bft/msg_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/mod.rs
@@ -1,20 +1,12 @@
 //! A module to manage the `febft` message log.
 
-use std::path::Path;
-use std::sync::Arc;
-
 use atlas_common::error::*;
-use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::SeqNo;
-use atlas_communication::message::{Header, StoredMessage};
-use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::ApplicationData;
+use atlas_communication::message::Header;
 use atlas_core::messages::RequestMessage;
-use atlas_core::state_transfer::Checkpoint;
+use atlas_execution::serialize::ApplicationData;
 
-use crate::bft::message::ConsensusMessage;
-use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::msg_log::decided_log::Log;
 use crate::bft::msg_log::decisions::DecisionLog;
 
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index 1b41afeb..edda4232 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -10,11 +10,9 @@ use atlas_common::channel::TryRecvError;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::threadpool;
-use atlas_communication::protocol_node::ProtocolNetworkNode;
-use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
+use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::request_pre_processing::{BatchOutput, PreProcessorOutputMessage};
-use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::Timeouts;
 use atlas_execution::app::UnorderedBatch;
 use atlas_execution::ExecutorHandle;
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index e978f765..968d2a80 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -20,25 +20,23 @@ use atlas_common::crypto::hash::Digest;
 use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_pop_message, tbo_queue_message};
-use atlas_common::threadpool::join;
 use atlas_communication::message::{Header, StoredMessage, WireMessage};
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
-use atlas_core::messages::{ClientRqInfo, StoredRequestMessage, SystemMessage};
+use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::ReconfigurationAttemptResult;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::request_pre_processing::RequestPreProcessor;
-use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_execution::serialize::ApplicationData;
 
-use crate::bft::PBFT;
 use crate::bft::consensus::Consensus;
 use crate::bft::message::{ConsensusMessageKind, FwdConsensusMessage, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::msg_log::decided_log::Log;
 use crate::bft::msg_log::decisions::{CollectData, Proof, StoredConsensusMessage, ViewDecisionPair};
+use crate::bft::PBFT;
 use crate::bft::sync::view::ViewInfo;
 
 use self::{follower_sync::FollowerSynchronizer, replica_sync::ReplicaSynchronizer};
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 6d65d667..72d48567 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -4,31 +4,27 @@
 //! leader is elected.
 
 use std::cell::Cell;
-
 use std::marker::PhantomData;
-
 use std::time::{Duration, Instant};
+
 use log::{debug, error, info};
-use num_traits::real::Real;
+
 use atlas_common::collections;
 use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{Orderable};
-use atlas_communication::message::{NetworkMessageKind};
+use atlas_common::ordering::Orderable;
 use atlas_communication::protocol_node::ProtocolNetworkNode;
-use atlas_execution::serialize::ApplicationData;
-use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, RequestMessage, StoredRequestMessage, SystemMessage};
+use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_core::reconfiguration_protocol::QuorumJoinCert;
+use atlas_core::persistent_log::OrderingProtocolLog;
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
-use atlas_core::serialize::{LogTransferMessage, ReconfigurationProtocolMessage, StateTransferMessage};
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, TimeoutPhase, Timeouts};
+use atlas_execution::serialize::ApplicationData;
 use atlas_metrics::metrics::{metric_duration, metric_increment};
-use crate::bft::consensus::Consensus;
 
+use crate::bft::consensus::Consensus;
+use crate::bft::message::{ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
-use crate::bft::metric::{SYNC_BATCH_RECEIVED_ID, SYNC_FORWARDED_COUNT_ID, SYNC_FORWARDED_REQUESTS_ID, SYNC_STOPPED_COUNT_ID, SYNC_STOPPED_REQUESTS_ID, SYNC_WATCH_REQUESTS_ID};
+use crate::bft::metric::{SYNC_BATCH_RECEIVED_ID, SYNC_STOPPED_COUNT_ID, SYNC_STOPPED_REQUESTS_ID, SYNC_WATCH_REQUESTS_ID};
 use crate::bft::msg_log::decided_log::Log;
 use crate::bft::msg_log::decisions::{CollectData, StoredConsensusMessage};
 use crate::bft::PBFT;
diff --git a/febft-pbft-consensus/src/bft/sync/view/mod.rs b/febft-pbft-consensus/src/bft/sync/view/mod.rs
index 27c9c05d..b329e7d4 100644
--- a/febft-pbft-consensus/src/bft/sync/view/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/view/mod.rs
@@ -12,8 +12,8 @@ use atlas_common::crypto::hash::Digest;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
+use atlas_core::ordering_protocol::networking::serialize::NetworkView;
 use atlas_execution::system_params::SystemParams;
-use atlas_core::serialize::NetworkView;
 
 /// This struct contains information related with an
 /// active `febft` view.
diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 3e6661a0..7e1fe874 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -8,8 +8,8 @@ use std::time::{Duration, Instant};
 use log::{debug, error, info, warn};
 #[cfg(feature = "serialize_serde")]
 use serde::{Deserialize, Serialize};
-use atlas_common::channel::ChannelSyncTx;
 
+use atlas_common::channel::ChannelSyncTx;
 use atlas_common::collections;
 use atlas_common::collections::HashMap;
 use atlas_common::crypto::hash::Digest;
@@ -17,19 +17,18 @@ use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_communication::message::{Header, NetworkMessageKind, StoredMessage};
+use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
-use atlas_execution::app::{Application, Reply, Request};
-use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::ApplicationData;
-use atlas_core::messages::{StateTransfer, SystemMessage};
-use atlas_core::ordering_protocol::{ExecutionResult, OrderingProtocol, SerProof, View};
-use atlas_core::persistent_log::{MonolithicStateLog, PersistableStateTransferProtocol, OperationMode};
-use atlas_core::serialize::{LogTransferMessage, NetworkView, OrderingProtocolMessage, ServiceMsg, StatefulOrderProtocolMessage, StateTransferMessage};
+use atlas_core::messages::StateTransfer;
+use atlas_core::ordering_protocol::{ExecutionResult, OrderingProtocol};
+use atlas_core::ordering_protocol::networking::serialize::NetworkView;
+use atlas_core::persistent_log::{MonolithicStateLog, OperationMode, PersistableStateTransferProtocol};
 use atlas_core::state_transfer::{Checkpoint, CstM, StateTransferProtocol, STResult, STTimeoutResult};
 use atlas_core::state_transfer::monolithic_state::MonolithicStateTransfer;
 use atlas_core::state_transfer::networking::StateTransferSendNode;
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, Timeouts};
+use atlas_execution::app::Application;
+use atlas_execution::serialize::ApplicationData;
 use atlas_execution::state::monolithic_state::{InstallStateMessage, MonolithicState};
 use atlas_metrics::metrics::metric_duration;
 
diff --git a/febft-state-transfer/src/message/serialize/capnp/mod.rs b/febft-state-transfer/src/message/serialize/capnp/mod.rs
index b2ba29fa..e294ed2b 100644
--- a/febft-state-transfer/src/message/serialize/capnp/mod.rs
+++ b/febft-state-transfer/src/message/serialize/capnp/mod.rs
@@ -1,6 +1,5 @@
 use atlas_execution::serialize::ApplicationData;
 use atlas_common::error::*;
-use atlas_core::state_transfer::StatefulOrderProtocol;
 use atlas_execution::state::divisible_state::DivisibleState;
 use crate::message::CstMessage;
 
diff --git a/febft-state-transfer/src/message/serialize/mod.rs b/febft-state-transfer/src/message/serialize/mod.rs
index 467d64da..a19f146f 100644
--- a/febft-state-transfer/src/message/serialize/mod.rs
+++ b/febft-state-transfer/src/message/serialize/mod.rs
@@ -1,35 +1,27 @@
-#[cfg(feature = "serialize_capnp")]
-mod capnp;
-
 use std::marker::PhantomData;
 use std::sync::Arc;
-use std::time::Instant;
+
 use atlas_communication::message::Header;
-use atlas_communication::message_signing::NetworkMessageSignatureVerifier;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
-use atlas_communication::serialize::Serializable;
-use atlas_execution::serialize::ApplicationData;
-use atlas_core::serialize::{InternallyVerifiable, OrderingProtocolMessage, StatefulOrderProtocolMessage, StateTransferMessage};
-use atlas_core::state_transfer::{StateTransferProtocol};
+use atlas_core::state_transfer::networking::serialize::StateTransferMessage;
+use atlas_core::state_transfer::networking::signature_ver::StateTransferVerificationHelper;
 use atlas_execution::state::monolithic_state::MonolithicState;
-use crate::CollabStateTransfer;
+
 use crate::message::CstMessage;
 
-pub struct CSTMsg<S: MonolithicState>(PhantomData<(S)>);
+#[cfg(feature = "serialize_capnp")]
+mod capnp;
 
-impl<S: MonolithicState> InternallyVerifiable<CstMessage<S>> for CSTMsg<S> {
-    fn verify_internal_message<M, SV, NI>(network_info: &Arc<NI>, header: &Header, msg: &CstMessage<S>) -> atlas_common::error::Result<bool>
-        where M: Serializable,
-              SV: NetworkMessageSignatureVerifier<M, NI>,
-              NI: NetworkInformationProvider {
-        Ok(true)
-    }
-}
+pub struct CSTMsg<S: MonolithicState>(PhantomData<(S)>);
 
 impl<S: MonolithicState> StateTransferMessage for CSTMsg<S> {
-
     type StateTransferMessage = CstMessage<S>;
 
+    fn verify_state_message<NI, SVH>(network_info: &Arc<NI>, header: &Header, message: Self::StateTransferMessage) -> atlas_common::error::Result<(bool, Self::StateTransferMessage)>
+        where NI: NetworkInformationProvider, SVH: StateTransferVerificationHelper {
+        todo!()
+    }
+
     #[cfg(feature = "serialize_capnp")]
     fn serialize_capnp(builder: atlas_capnp::cst_messages_capnp::cst_message::Builder, msg: &Self::StateTransferMessage) -> atlas_common::error::Result<()> {
         todo!()

From fc9d87bac0d7a18e59c3fadebe4681dbf2c43741 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Fri, 25 Aug 2023 19:27:14 +0100
Subject: [PATCH 46/80] Compound verification for protocol messages, assuring
 all contained forwarded messages are not forged.

---
 .../src/bft/message/serialize/mod.rs          | 93 ++++++++++++++++++-
 febft-pbft-consensus/src/bft/sync/mod.rs      |  4 +
 2 files changed, 96 insertions(+), 1 deletion(-)

diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index 8468df4b..9c7f247d 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -6,6 +6,7 @@
 //! as [Cap'n'Proto](https://capnproto.org/capnp-tool.html), but these are
 //! expected to be implemented by the user.
 
+use std::fmt::Debug;
 use std::io::{Read, Write};
 use std::marker::PhantomData;
 use std::sync::Arc;
@@ -19,6 +20,7 @@ use atlas_communication::message::Header;
 use atlas_communication::message_signing::NetworkMessageSignatureVerifier;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_communication::serialize::Serializable;
+use atlas_core::messages::SystemMessage;
 use atlas_core::ordering_protocol::networking::serialize::{OrderingProtocolMessage, StatefulOrderProtocolMessage};
 use atlas_core::ordering_protocol::networking::signature_ver::OrderProtocolSignatureVerificationHelper;
 use atlas_core::persistent_log::PersistableOrderProtocol;
@@ -145,7 +147,96 @@ impl<D> OrderingProtocolMessage for PBFTConsensus<D>
     type ProofMetadata = ProofMetadata;
 
     fn verify_order_protocol_message<NI, OPVH, D2>(network_info: &NI, header: &Header, message: Self::ProtocolMessage) -> Result<(bool, Self::ProtocolMessage)> where NI: NetworkInformationProvider, OPVH: OrderProtocolSignatureVerificationHelper<D2, Self, NI>, D2: ApplicationData, Self: Sized {
-        todo!()
+        match &message {
+            PBFTMessage::Consensus(consensus) => {
+                match consensus.kind() {
+                    ConsensusMessageKind::PrePrepare(requests) => {
+                        for request in requests {
+                            let (result, _) = OPVH::verify_request_message(network_info, request.header(), request.message().clone())?;
+
+                            if !result {
+                                return Ok((false, message));
+                            }
+                        }
+
+                        Ok((true, message))
+                    }
+                    ConsensusMessageKind::Prepare(digest) => {
+                        Ok((true, message))
+                    }
+                    ConsensusMessageKind::Commit(digest) => {
+                        Ok((true, message))
+                    }
+                }
+            }
+            PBFTMessage::ViewChange(view_change) => {
+                match view_change.kind() {
+                    ViewChangeMessageKind::Stop(timed_out_req) => {
+                        for client_rq in timed_out_req {
+                            let (result, _) = OPVH::verify_request_message(network_info, client_rq.header(), client_rq.message().clone())?;
+
+                            if !result {
+                                return Ok((false, message));
+                            }
+                        }
+
+                        Ok((true, message))
+                    }
+                    ViewChangeMessageKind::StopQuorumJoin(_) => {
+                        Ok((true, message))
+                    }
+                    ViewChangeMessageKind::StopData(collect_data) => {
+                        if let Some(proof) = &collect_data.last_proof {
+                            for pre_prepare in proof.pre_prepares() {
+                                let (result, _) = OPVH::verify_protocol_message(network_info, pre_prepare.header(), PBFTMessage::Consensus(pre_prepare.message().clone()))?;
+
+                                if !result {
+                                    return Ok((false, message));
+                                }
+                            }
+
+                            for prepare in proof.prepares() {
+                                let (result, _) = OPVH::verify_protocol_message(network_info, prepare.header(), PBFTMessage::Consensus(prepare.message().clone()))?;
+
+                                if !result {
+                                    return Ok((false, message));
+                                }
+                            }
+
+                            for commit in proof.commits() {
+                                let (result, _) = OPVH::verify_protocol_message(network_info, commit.header(), PBFTMessage::Consensus(commit.message().clone()))?;
+
+                                if !result {
+                                    return Ok((false, message));
+                                }
+                            }
+                        }
+
+                        Ok((true, message))
+                    }
+                    ViewChangeMessageKind::Sync(leader_collects) => {
+                        let (result, _) = OPVH::verify_protocol_message(network_info, leader_collects.message().header(), PBFTMessage::Consensus(leader_collects.message().consensus().clone()))?;
+
+                        if !result {
+                            return Ok((false, message));
+                        }
+
+                        for collect in leader_collects.collects() {
+
+                            let (result, _) = OPVH::verify_protocol_message(network_info, collect.header(), PBFTMessage::ViewChange(collect.message().clone()))?;
+
+                            if !result {
+                                return Ok((false, message));
+                            }
+
+                        }
+
+                        Ok((true, message))
+                    }
+                }
+            }
+            PBFTMessage::ObserverMessage(_) => Ok((true, message))
+        }
     }
 
     #[cfg(feature = "serialize_capnp")]
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 968d2a80..6b0e512a 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -126,6 +126,10 @@ impl<O> LeaderCollects<O> {
         &self.proposed
     }
 
+    pub fn collects(&self) -> &Vec<StoredMessage<ViewChangeMessage<O>>> {
+        &self.collects
+    }
+
     /// Gives up ownership of the inner values of this `LeaderCollects`.
     pub fn into_inner(
         self,

From 33b1417fcfa2173eff127966ba60ac037158d921 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 28 Aug 2023 17:20:13 +0100
Subject: [PATCH 47/80] Fixed issues with message signature verification (had
 to include some more generics to have the necessary accesses)

---
 febft-pbft-consensus/src/bft/config/mod.rs    |  4 +-
 .../src/bft/consensus/decision/mod.rs         |  2 +-
 febft-pbft-consensus/src/bft/consensus/mod.rs |  6 +-
 .../src/bft/message/serialize/mod.rs          | 71 +++++++++++--------
 febft-pbft-consensus/src/bft/mod.rs           | 56 +++++++--------
 .../src/bft/msg_log/decided_log/mod.rs        | 12 ++--
 febft-pbft-consensus/src/bft/sync/mod.rs      | 14 ++--
 .../src/bft/sync/replica_sync/mod.rs          |  2 +-
 8 files changed, 89 insertions(+), 78 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/config/mod.rs b/febft-pbft-consensus/src/bft/config/mod.rs
index 3e67cf37..6efdd758 100644
--- a/febft-pbft-consensus/src/bft/config/mod.rs
+++ b/febft-pbft-consensus/src/bft/config/mod.rs
@@ -11,7 +11,7 @@ pub struct PBFTConfig<D>
     where D: ApplicationData {
     pub node_id: NodeId,
     // pub observer_handle: ObserverHandle,
-    pub follower_handle: Option<FollowerHandle<PBFTConsensus<D>>>,
+    pub follower_handle: Option<FollowerHandle<D, PBFTConsensus<D>>>,
     pub view: ViewInfo,
     pub timeout_dur: Duration,
     pub proposer_config: ProposerConfig,
@@ -20,7 +20,7 @@ pub struct PBFTConfig<D>
 
 impl<D: ApplicationData + 'static, > PBFTConfig<D> {
     pub fn new(node_id: NodeId,
-               follower_handle: Option<FollowerHandle<PBFTConsensus<D>>>,
+               follower_handle: Option<FollowerHandle<D, PBFTConsensus<D>>>,
                view: ViewInfo, timeout_dur: Duration,
                watermark: u32, proposer_config: ProposerConfig) -> Self {
         Self {
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 1f9dbbfe..fa3a2652 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -266,7 +266,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                                log: &mut Log<D, PL>,
                                node: &Arc<NT>) -> Result<DecisionStatus>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D>> {
+              PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let view = synchronizer.view();
 
         return match self.phase {
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 4f934d72..66008ff9 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -367,7 +367,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
                                log: &mut Log<D, PL>,
                                node: &Arc<NT>) -> Result<ConsensusStatus>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D>>, {
+              PL: OrderingProtocolLog<D, PBFTConsensus<D>>, {
         let message_seq = message.sequence_number();
 
         let view_seq = message.view();
@@ -696,7 +696,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
                               view: &ViewInfo,
                               proof: Proof<D::Request>,
                               log: &mut Log<D, PL>) -> Result<ProtocolConsensusDecision<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
 
         // If this is successful, it means that we are all caught up and can now start executing the
         // batch
@@ -802,7 +802,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
         node: &Arc<NT>,
     ) where
         NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-        PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         //Prepare the algorithm as we are already entering this phase
 
         self.install_view(new_view);
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index 9c7f247d..615508d9 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -138,7 +138,7 @@ impl<D> PBFTConsensus<D> where D: ApplicationData {
     }
 }
 
-impl<D> OrderingProtocolMessage for PBFTConsensus<D>
+impl<D> OrderingProtocolMessage<D> for PBFTConsensus<D>
     where D: ApplicationData, {
     type ViewInfo = ViewInfo;
     type ProtocolMessage = PBFTMessage<D::Request>;
@@ -146,7 +146,7 @@ impl<D> OrderingProtocolMessage for PBFTConsensus<D>
     type Proof = Proof<D::Request>;
     type ProofMetadata = ProofMetadata;
 
-    fn verify_order_protocol_message<NI, OPVH, D2>(network_info: &NI, header: &Header, message: Self::ProtocolMessage) -> Result<(bool, Self::ProtocolMessage)> where NI: NetworkInformationProvider, OPVH: OrderProtocolSignatureVerificationHelper<D2, Self, NI>, D2: ApplicationData, Self: Sized {
+    fn verify_order_protocol_message<NI, OPVH>(network_info: &Arc<NI>, header: &Header, message: Self::ProtocolMessage) -> Result<(bool, Self::ProtocolMessage)> where NI: NetworkInformationProvider, OPVH: OrderProtocolSignatureVerificationHelper<D, Self, NI>, Self: Sized {
         match &message {
             PBFTMessage::Consensus(consensus) => {
                 match consensus.kind() {
@@ -186,31 +186,7 @@ impl<D> OrderingProtocolMessage for PBFTConsensus<D>
                         Ok((true, message))
                     }
                     ViewChangeMessageKind::StopData(collect_data) => {
-                        if let Some(proof) = &collect_data.last_proof {
-                            for pre_prepare in proof.pre_prepares() {
-                                let (result, _) = OPVH::verify_protocol_message(network_info, pre_prepare.header(), PBFTMessage::Consensus(pre_prepare.message().clone()))?;
-
-                                if !result {
-                                    return Ok((false, message));
-                                }
-                            }
-
-                            for prepare in proof.prepares() {
-                                let (result, _) = OPVH::verify_protocol_message(network_info, prepare.header(), PBFTMessage::Consensus(prepare.message().clone()))?;
-
-                                if !result {
-                                    return Ok((false, message));
-                                }
-                            }
-
-                            for commit in proof.commits() {
-                                let (result, _) = OPVH::verify_protocol_message(network_info, commit.header(), PBFTMessage::Consensus(commit.message().clone()))?;
-
-                                if !result {
-                                    return Ok((false, message));
-                                }
-                            }
-                        }
+                        if let Some(proof) = &collect_data.last_proof {}
 
                         Ok((true, message))
                     }
@@ -222,13 +198,11 @@ impl<D> OrderingProtocolMessage for PBFTConsensus<D>
                         }
 
                         for collect in leader_collects.collects() {
-
                             let (result, _) = OPVH::verify_protocol_message(network_info, collect.header(), PBFTMessage::ViewChange(collect.message().clone()))?;
 
                             if !result {
                                 return Ok((false, message));
                             }
-
                         }
 
                         Ok((true, message))
@@ -239,6 +213,35 @@ impl<D> OrderingProtocolMessage for PBFTConsensus<D>
         }
     }
 
+    fn verify_proof<NI, OPVH>(network_info: &Arc<NI>, proof: Self::Proof) -> Result<(bool, Self::Proof)>
+        where NI: NetworkInformationProvider, OPVH: OrderProtocolSignatureVerificationHelper<D, Self, NI>, Self: Sized {
+        for pre_prepare in proof.pre_prepares() {
+            let (result, _) = OPVH::verify_protocol_message(network_info, pre_prepare.header(), PBFTMessage::Consensus(pre_prepare.message().clone()))?;
+
+            if !result {
+                return Ok((false, proof));
+            }
+        }
+
+        for prepare in proof.prepares() {
+            let (result, _) = OPVH::verify_protocol_message(network_info, prepare.header(), PBFTMessage::Consensus(prepare.message().clone()))?;
+
+            if !result {
+                return Ok((false, proof));
+            }
+        }
+
+        for commit in proof.commits() {
+            let (result, _) = OPVH::verify_protocol_message(network_info, commit.header(), PBFTMessage::Consensus(commit.message().clone()))?;
+
+            if !result {
+                return Ok((false, proof));
+            }
+        }
+
+        return Ok((true, proof));
+    }
+
     #[cfg(feature = "serialize_capnp")]
     fn serialize_capnp(builder: atlas_capnp::consensus_messages_capnp::protocol_message::Builder, msg: &Self::ProtocolMessage) -> Result<()> {
         capnp::serialize_message::<D>(builder, msg)
@@ -270,10 +273,18 @@ impl<D> OrderingProtocolMessage for PBFTConsensus<D>
     }
 }
 
-impl<D> StatefulOrderProtocolMessage for PBFTConsensus<D>
+impl<D, OP> StatefulOrderProtocolMessage<D, OP> for PBFTConsensus<D>
     where D: ApplicationData + 'static {
     type DecLog = DecisionLog<D::Request>;
 
+    fn verify_decision_log<NI, OPVH>(network_info: &Arc<NI>, dec_log: Self::DecLog) -> Result<(bool, Self::DecLog)>
+        where NI: NetworkInformationProvider,
+              D: ApplicationData,
+              OP: OrderingProtocolMessage<D>,
+              OPVH: OrderProtocolSignatureVerificationHelper<D, OP, NI>, {
+        Ok((false, dec_log))
+    }
+
     #[cfg(feature = "serialize_capnp")]
     fn serialize_declog_capnp(builder: atlas_capnp::cst_messages_capnp::dec_log::Builder, msg: &Self::DecLog) -> Result<()> {
         todo!()
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index f1ec36f2..91b34a58 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -56,7 +56,7 @@ pub mod metric;
 // The types responsible for this protocol
 pub type PBFT<D> = PBFTConsensus<D>;
 // The message type for this consensus protocol
-pub type SysMsg<D> = <PBFTConsensus<D> as OrderingProtocolMessage>::ProtocolMessage;
+pub type SysMsg<D> = <PBFTConsensus<D> as OrderingProtocolMessage<D>>::ProtocolMessage;
 
 #[derive(Clone, PartialEq, Eq, Debug)]
 /// Which phase of the consensus algorithm are we currently executing
@@ -141,12 +141,12 @@ impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
         Self::initialize_protocol(config, args, None)
     }
 
-    fn view(&self) -> View<Self::Serialization> {
+    fn view(&self) -> View<D, Self::Serialization> {
         self.synchronizer.view()
     }
 
     fn handle_off_ctx_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>)
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let (header, message) = message.into_inner();
 
         match message.into_inner() {
@@ -182,7 +182,7 @@ impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
     }
 
     fn poll(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         trace!("{:?} // Polling {:?}", self.node.id(), self.phase);
 
         match self.phase {
@@ -196,7 +196,7 @@ impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
     }
 
     fn process_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         match self.phase {
             ConsensusPhase::NormalPhase => {
                 self.update_normal_phase(message)
@@ -207,12 +207,12 @@ impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
         }
     }
 
-    fn sequence_number_with_proof(&self) -> Result<Option<(SeqNo, SerProof<Self::Serialization>)>> {
+    fn sequence_number_with_proof(&self) -> Result<Option<(SeqNo, SerProof<D, Self::Serialization>)>> {
         Ok(self.message_log.last_proof(self.synchronizer.view().f())
             .map(|p| (p.sequence_number(), p)))
     }
 
-    fn verify_sequence_number(&self, seq_no: SeqNo, proof: &SerProof<Self::Serialization>) -> Result<bool> {
+    fn verify_sequence_number(&self, seq_no: SeqNo, proof: &SerProof<D, Self::Serialization>) -> Result<bool> {
         let proof: &Proof<D::Request> = proof;
 
         //TODO: Verify the proof
@@ -227,7 +227,7 @@ impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
     }
 
     fn handle_timeout(&mut self, timeout: Vec<RqTimeout>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         if self.consensus.is_catching_up() {
             warn!("{:?} // Ignoring timeouts while catching up", self.node.id());
 
@@ -331,7 +331,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     }
 
     fn poll_sync_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
 
         // retrieve a view change message to be processed
         let poll_result = self.synchronizer.poll();
@@ -371,7 +371,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     }
 
     fn poll_normal_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         // check if we have STOP messages to be processed,
         // and update our phase when we start installing
         // the new view
@@ -435,7 +435,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     }
 
     fn update_sync_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let (header, protocol) = message.into_inner();
 
         match protocol.into_inner() {
@@ -479,7 +479,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     }
 
     fn update_normal_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let (header, protocol) = message.into_inner();
 
         match protocol.into_inner() {
@@ -522,7 +522,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         header: Header,
         message: ConsensusMessage<D::Request>,
     ) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let seq = self.consensus.sequence_number();
 
         // debug!(
@@ -590,7 +590,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     /// Advance the sync phase of the algorithm
     fn adv_sync(&mut self, header: Header,
                 message: ViewChangeMessage<D::Request>) -> SyncPhaseRes<D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let status = self.synchronizer.process_message(
             header,
             message,
@@ -656,9 +656,9 @@ impl<D, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL
     }
 
     fn install_state(&mut self,
-                     view_info: View<Self::Serialization>,
-                     dec_log: DecLog<Self::StateSerialization>) -> Result<Vec<D::Request>>
-        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
+                     view_info: View<D, Self::Serialization>,
+                     dec_log: DecLog<D, Self::Serialization, Self::StateSerialization>) -> Result<Vec<D::Request>>
+        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>> {
         info!("{:?} // Installing decision log with Seq No {:?} and View {:?}", self.node.id(),
                 dec_log.sequence_number(), view_info);
 
@@ -690,12 +690,12 @@ impl<D, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL
         Ok(res)
     }
 
-    fn snapshot_log(&mut self) -> Result<(View<Self::Serialization>, DecLog<Self::StateSerialization>)> {
+    fn snapshot_log(&mut self) -> Result<(View<D, Self::Serialization>, DecLog<D, Self::Serialization, Self::StateSerialization>)> {
         self.message_log.snapshot(self.synchronizer.view())
     }
 
-    fn current_log(&self) -> Result<&DecLog<Self::StateSerialization>>
-        where PL: StatefulOrderingProtocolLog<Self::Serialization, Self::StateSerialization> {
+    fn current_log(&self) -> Result<&DecLog<D, Self::Serialization, Self::StateSerialization>>
+        where PL: StatefulOrderingProtocolLog<D, Self::Serialization, Self::StateSerialization> {
         Ok(self.message_log.decision_log())
     }
 
@@ -703,7 +703,7 @@ impl<D, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL
         self.message_log.finalize_checkpoint(seq_no)
     }
 
-    fn get_proof(&self, seq: SeqNo) -> Result<Option<SerProof<Self::Serialization>>> {
+    fn get_proof(&self, seq: SeqNo) -> Result<Option<SerProof<D, Self::Serialization>>> {
         todo!()
     }
 }
@@ -770,7 +770,7 @@ const CF_PRE_PREPARES: &str = "PRE_PREPARES";
 const CF_PREPARES: &str = "PREPARES";
 const CF_COMMIT: &str = "COMMITS";
 
-impl<D, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus<D>> for PBFTOrderProtocol<D, NT, PL>
+impl<D, NT, PL> PersistableOrderProtocol<D, PBFTConsensus<D>, PBFTConsensus<D>> for PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>>, PL: Clone {
     fn message_types() -> Vec<&'static str> {
@@ -781,7 +781,7 @@ impl<D, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus<D>> for
         ]
     }
 
-    fn get_type_for_message(msg: &LoggableMessage<PBFTConsensus<D>>) -> Result<&'static str> {
+    fn get_type_for_message(msg: &LoggableMessage<D, PBFTConsensus<D>>) -> Result<&'static str> {
         match msg.kind() {
             ConsensusMessageKind::PrePrepare(_) => {
                 Ok(CF_PRE_PREPARES)
@@ -795,7 +795,7 @@ impl<D, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus<D>> for
         }
     }
 
-    fn init_proof_from(metadata: SerProofMetadata<PBFTConsensus<D>>, messages: Vec<StoredMessage<LoggableMessage<PBFTConsensus<D>>>>) -> SerProof<PBFTConsensus<D>> {
+    fn init_proof_from(metadata: SerProofMetadata<D, PBFTConsensus<D>>, messages: Vec<StoredMessage<LoggableMessage<D, PBFTConsensus<D>>>>) -> SerProof<D, PBFTConsensus<D>> {
         let mut pre_prepares = Vec::with_capacity(messages.len() / 2);
         let mut prepares = Vec::with_capacity(messages.len() / 2);
         let mut commits = Vec::with_capacity(messages.len() / 2);
@@ -817,11 +817,11 @@ impl<D, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus<D>> for
         Proof::new(metadata, pre_prepares, prepares, commits)
     }
 
-    fn init_dec_log(proofs: Vec<SerProof<PBFTConsensus<D>>>) -> DecLog<PBFTConsensus<D>> {
+    fn init_dec_log(proofs: Vec<SerProof<D, PBFTConsensus<D>>>) -> DecLog<D, PBFTConsensus<D>, PBFTConsensus<D>> {
         DecisionLog::from_proofs(proofs)
     }
 
-    fn decompose_proof(proof: &SerProof<PBFTConsensus<D>>) -> (&SerProofMetadata<PBFTConsensus<D>>, Vec<&StoredMessage<LoggableMessage<PBFTConsensus<D>>>>) {
+    fn decompose_proof(proof: &SerProof<D, PBFTConsensus<D>>) -> (&SerProofMetadata<D, PBFTConsensus<D>>, Vec<&StoredMessage<LoggableMessage<D, PBFTConsensus<D>>>>) {
         let mut messages = Vec::new();
 
         for message in proof.pre_prepares() {
@@ -839,7 +839,7 @@ impl<D, NT, PL> PersistableOrderProtocol<PBFTConsensus<D>, PBFTConsensus<D>> for
         (proof.metadata(), messages)
     }
 
-    fn decompose_dec_log(proofs: &DecLog<PBFTConsensus<D>>) -> Vec<&SerProof<PBFTConsensus<D>>> {
+    fn decompose_dec_log(proofs: &DecLog<D, PBFTConsensus<D>, PBFTConsensus<D>>) -> Vec<&SerProof<D, PBFTConsensus<D>>> {
         proofs.proofs().iter().collect()
     }
 }
@@ -848,7 +848,7 @@ impl<D, NT, PL, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, NT,
     where D: ApplicationData + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-          PL: OrderingProtocolLog<PBFTConsensus<D>> {
+          PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
     fn attempt_quorum_node_join(&mut self, joining_node: NodeId) -> Result<ReconfigurationAttemptResult> {
         let result = self.synchronizer.start_join_quorum(joining_node, &*self.node, &self.timeouts, &self.message_log);
 
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index 65c58f30..8b62a0f1 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -48,7 +48,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     ///
     /// FIXME: The view initialization might have to be changed if we want to introduce reconfiguration
     pub fn read_current_state(&self, n: usize, f: usize) -> Result<Option<(ViewInfo, DecisionLog<D::Request>)>>
-        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
+        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>> {
         let option = self.persistent_log.read_state(OperationMode::BlockingSync)?;
 
         if let Some((view, dec_log)) = option {
@@ -74,7 +74,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     pub fn insert_consensus(
         &mut self,
         consensus_msg: StoredConsensusMessage<D::Request>,
-    ) where PL: OrderingProtocolLog<PBFTConsensus<D>>,
+    ) where PL: OrderingProtocolLog<D, PBFTConsensus<D>>,
     {
         if let Err(err) = self
             .persistent_log
@@ -89,7 +89,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// which contains all of the collects
     /// If we are missing the request determined by the
     pub fn install_proof(&mut self, seq: SeqNo, proof: Proof<D::Request>) -> Result<ProtocolConsensusDecision<D::Request>>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let batch_execution_info = ProtocolConsensusDecision::from(&proof);
 
         if let Some(decision) = self.decision_log().last_decision() {
@@ -114,7 +114,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
 
     /// Clear the occurrences of a seq no from the decision log
     pub fn clear_last_occurrence(&mut self, seq: SeqNo)
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         if let Err(err) = self.persistent_log.write_invalidate(OperationMode::NonBlockingSync(None), seq) {
             error!("Failed to invalidate last occurrence {:?}", err);
         }
@@ -122,7 +122,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
 
     /// Update the log state, received from the CST protocol.
     pub fn install_state(&mut self, view: ViewInfo, dec_log: DecisionLog<D::Request>)
-        where PL: StatefulOrderingProtocolLog<PBFTConsensus<D>, PBFTConsensus<D>> {
+        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>> {
 
         //Replace the log
         self.dec_log = dec_log.clone();
@@ -158,7 +158,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// Register that all the batches for a given decision have already been received
     /// Basically persists the metadata for a given consensus num
     pub fn all_batches_received(&mut self, metadata: ProofMetadata)
-        where PL: OrderingProtocolLog<PBFTConsensus<D>> {
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         self.persistent_log.write_proof_metadata(OperationMode::NonBlockingSync(None),
                                                  metadata).unwrap();
     }
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 6b0e512a..79d20d5f 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -692,7 +692,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D>>
+              PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
         debug!("{:?} // Processing view change message {:?} in phase {:?} from {:?}",
                node.id(),
@@ -1332,7 +1332,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         node: &Arc<NT>,
     ) -> Option<SynchronizerStatus<D::Request>>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D>>
+              PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
 
         let state = self.finalize_state.borrow_mut().take()?;
@@ -1356,7 +1356,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     /// of the system
     pub fn start_join_quorum<NT, PL>(&self, joining_node: NodeId, node: &NT, timeouts: &Timeouts, log: &Log<D, PL>) -> SyncReconfigurationResult
         where NT: OrderProtocolSendNode<D, PBFT<D>>,
-              PL: OrderingProtocolLog<PBFTConsensus<D>> {
+              PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let current_view = self.view();
 
         info!("{:?} // Starting the quorum join procedure for node {:?}", node.id(), joining_node);
@@ -1453,7 +1453,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                                             timeouts: &Timeouts,
                                             _log: &Log<D, PL>, )
         where NT: OrderProtocolSendNode<D, PBFT<D>>,
-              PL: OrderingProtocolLog<PBFTConsensus<D>>
+              PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
         debug!("Beginning quorum view change with certificate {} at phase {:?}",  join_cert.is_some(), self.phase.get());
 
@@ -1509,7 +1509,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         timeouts: &Timeouts,
         _log: &Log<D, PL>,
     ) where NT: OrderProtocolSendNode<D, PBFT<D>>,
-            PL: OrderingProtocolLog<PBFTConsensus<D>>
+            PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
         match (self.phase.get(), &timed_out) {
             // we have received STOP messages from peer nodes,
@@ -1578,7 +1578,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         _normalized_collects: Vec<Option<&CollectData<D::Request>>>,
         log: &Log<D, PL>,
     ) -> FinalizeStatus<D::Request>
-        where PL: OrderingProtocolLog<PBFTConsensus<D>>
+        where PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
         let last_executed_cid = proof.as_ref().map(|p| p.sequence_number()).unwrap_or(SeqNo::ZERO);
 
@@ -1613,7 +1613,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-              PL: OrderingProtocolLog<PBFTConsensus<D>>
+              PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
         let FinalizeState {
             curr_cid,
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 72d48567..23370566 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -67,7 +67,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         timeouts: &Timeouts,
         node: &NT,
     ) where NT: OrderProtocolSendNode<D, PBFT<D>>,
-            PL: OrderingProtocolLog<PBFTConsensus<D>> {
+            PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         // NOTE:
         // - install new view (i.e. update view seq no) (Done in the synchronizer)
         // - add requests from STOP into client requests

From 27b214bd27f51e936b1790c688cb8f2be08be768 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 18 Sep 2023 15:59:51 +0100
Subject: [PATCH 48/80] Abstracted the permissioned part of the consensus so
 that a permissionless system does not have to implement it.

---
 febft-pbft-consensus/src/bft/config/mod.rs    |  6 ++--
 .../src/bft/message/serialize/mod.rs          |  8 +++--
 febft-pbft-consensus/src/bft/mod.rs           | 31 ++++++++++++++-----
 .../src/bft/msg_log/decided_log/mod.rs        |  4 +--
 4 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/config/mod.rs b/febft-pbft-consensus/src/bft/config/mod.rs
index 6efdd758..1564b707 100644
--- a/febft-pbft-consensus/src/bft/config/mod.rs
+++ b/febft-pbft-consensus/src/bft/config/mod.rs
@@ -8,10 +8,10 @@ use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::sync::view::ViewInfo;
 
 pub struct PBFTConfig<D>
-    where D: ApplicationData {
+    where D: ApplicationData + 'static {
     pub node_id: NodeId,
     // pub observer_handle: ObserverHandle,
-    pub follower_handle: Option<FollowerHandle<D, PBFTConsensus<D>>>,
+    pub follower_handle: Option<FollowerHandle<D, PBFTConsensus<D>, PBFTConsensus<D>>>,
     pub view: ViewInfo,
     pub timeout_dur: Duration,
     pub proposer_config: ProposerConfig,
@@ -20,7 +20,7 @@ pub struct PBFTConfig<D>
 
 impl<D: ApplicationData + 'static, > PBFTConfig<D> {
     pub fn new(node_id: NodeId,
-               follower_handle: Option<FollowerHandle<D, PBFTConsensus<D>>>,
+               follower_handle: Option<FollowerHandle<D, PBFTConsensus<D>, PBFTConsensus<D>>>,
                view: ViewInfo, timeout_dur: Duration,
                watermark: u32, proposer_config: ProposerConfig) -> Self {
         Self {
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index 615508d9..fcdfe0ba 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -21,7 +21,7 @@ use atlas_communication::message_signing::NetworkMessageSignatureVerifier;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_communication::serialize::Serializable;
 use atlas_core::messages::SystemMessage;
-use atlas_core::ordering_protocol::networking::serialize::{OrderingProtocolMessage, StatefulOrderProtocolMessage};
+use atlas_core::ordering_protocol::networking::serialize::{OrderingProtocolMessage, PermissionedOrderingProtocolMessage, StatefulOrderProtocolMessage};
 use atlas_core::ordering_protocol::networking::signature_ver::OrderProtocolSignatureVerificationHelper;
 use atlas_core::persistent_log::PersistableOrderProtocol;
 use atlas_execution::serialize::ApplicationData;
@@ -140,7 +140,7 @@ impl<D> PBFTConsensus<D> where D: ApplicationData {
 
 impl<D> OrderingProtocolMessage<D> for PBFTConsensus<D>
     where D: ApplicationData, {
-    type ViewInfo = ViewInfo;
+
     type ProtocolMessage = PBFTMessage<D::Request>;
     type LoggableMessage = ConsensusMessage<D::Request>;
     type Proof = Proof<D::Request>;
@@ -273,6 +273,10 @@ impl<D> OrderingProtocolMessage<D> for PBFTConsensus<D>
     }
 }
 
+impl<D> PermissionedOrderingProtocolMessage for PBFTConsensus<D> where D: ApplicationData + 'static {
+    type ViewInfo = ViewInfo;
+}
+
 impl<D, OP> StatefulOrderProtocolMessage<D, OP> for PBFTConsensus<D>
     where D: ApplicationData + 'static {
     type DecLog = DecisionLog<D::Request>;
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 91b34a58..f5b85bfb 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -19,7 +19,7 @@ use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::serialize::Serializable;
 use atlas_core::messages::Protocol;
-use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, ProtocolConsensusDecision, SerProof, SerProofMetadata, View};
+use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, PermissionedOrderingProtocol, ProtocolConsensusDecision, SerProof, SerProofMetadata, View};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::networking::serialize::{NetworkView, OrderingProtocolMessage};
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::{ReconfigurableOrderProtocol, ReconfigurationAttemptResult};
@@ -141,9 +141,6 @@ impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
         Self::initialize_protocol(config, args, None)
     }
 
-    fn view(&self) -> View<D, Self::Serialization> {
-        self.synchronizer.view()
-    }
 
     fn handle_off_ctx_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>)
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
@@ -266,6 +263,24 @@ impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
     }
 }
 
+impl<D, NT, PL> PermissionedOrderingProtocol for PBFTOrderProtocol<D, NT, PL>
+    where D: ApplicationData + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
+          PL: Clone {
+
+    type PermissionedSerialization = PBFTConsensus<D>;
+
+    fn view(&self) -> View<Self::PermissionedSerialization> {
+        self.synchronizer.view()
+    }
+
+    fn install_view(&mut self, view: View<Self::PermissionedSerialization>) {
+        if self.synchronizer.received_view_from_state_transfer(view.clone()) {
+            self.consensus.install_view(&view)
+        }
+    }
+}
+
 impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
@@ -656,9 +671,9 @@ impl<D, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL
     }
 
     fn install_state(&mut self,
-                     view_info: View<D, Self::Serialization>,
+                     view_info: View<Self::PermissionedSerialization>,
                      dec_log: DecLog<D, Self::Serialization, Self::StateSerialization>) -> Result<Vec<D::Request>>
-        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>> {
+        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>, PBFTConsensus<D>> {
         info!("{:?} // Installing decision log with Seq No {:?} and View {:?}", self.node.id(),
                 dec_log.sequence_number(), view_info);
 
@@ -690,12 +705,12 @@ impl<D, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL
         Ok(res)
     }
 
-    fn snapshot_log(&mut self) -> Result<(View<D, Self::Serialization>, DecLog<D, Self::Serialization, Self::StateSerialization>)> {
+    fn snapshot_log(&mut self) -> Result<(View<Self::PermissionedSerialization>, DecLog<D, Self::Serialization, Self::StateSerialization>)> {
         self.message_log.snapshot(self.synchronizer.view())
     }
 
     fn current_log(&self) -> Result<&DecLog<D, Self::Serialization, Self::StateSerialization>>
-        where PL: StatefulOrderingProtocolLog<D, Self::Serialization, Self::StateSerialization> {
+        where PL: StatefulOrderingProtocolLog<D, Self::Serialization, Self::StateSerialization, Self::PermissionedSerialization> {
         Ok(self.message_log.decision_log())
     }
 
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index 8b62a0f1..5fc88739 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -48,7 +48,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     ///
     /// FIXME: The view initialization might have to be changed if we want to introduce reconfiguration
     pub fn read_current_state(&self, n: usize, f: usize) -> Result<Option<(ViewInfo, DecisionLog<D::Request>)>>
-        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>> {
+        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>, PBFTConsensus<D>> {
         let option = self.persistent_log.read_state(OperationMode::BlockingSync)?;
 
         if let Some((view, dec_log)) = option {
@@ -122,7 +122,7 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
 
     /// Update the log state, received from the CST protocol.
     pub fn install_state(&mut self, view: ViewInfo, dec_log: DecisionLog<D::Request>)
-        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>> {
+        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>, PBFTConsensus<D>> {
 
         //Replace the log
         self.dec_log = dec_log.clone();

From eb8589ee1b776328bfc8a19bd3254464b46023e9 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 28 Sep 2023 16:08:02 +0100
Subject: [PATCH 49/80] Fix dependencies, still have to fix imports, so this
 currently does not compile

---
 febft-pbft-consensus/Cargo.toml | 14 +++++++-------
 febft-state-transfer/Cargo.toml | 15 ++++++++-------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/febft-pbft-consensus/Cargo.toml b/febft-pbft-consensus/Cargo.toml
index 5f4472fb..8fe95b59 100644
--- a/febft-pbft-consensus/Cargo.toml
+++ b/febft-pbft-consensus/Cargo.toml
@@ -28,7 +28,7 @@ default = [
 ]
 
 serialize_serde = ["atlas-capnp", "serde", "serde_bytes", "bincode", "atlas-common/serialize_serde",
-    "atlas-execution/serialize_serde", "atlas-communication/serialize_serde", "atlas-core/serialize_serde"]
+    "atlas-smr-application/serialize_serde", "atlas-communication/serialize_serde", "atlas-core/serialize_serde"]
 serialize_capnp = ["atlas-capnp"]
 
 [dev-dependencies]
@@ -38,12 +38,12 @@ mimalloc = { version = "*", default-features = false }
 rand = {version = "0.8.5", features = ["small_rng"] }
 
 [dependencies]
-atlas-common = { path = "../../Atlas/atlas-common" }
-atlas-communication = { path = "../../Atlas/atlas-communication" }
-atlas-execution = { path = "../../Atlas/atlas-execution" }
-atlas-core = { path = "../../Atlas/atlas-core" }
-atlas-capnp = { path = "../../Atlas/atlas-capnp", optional = true }
-atlas-metrics = {path = "../../Atlas/atlas-metrics"}
+atlas-common = { path = "../../Atlas/Atlas-Common" }
+atlas-communication = { path = "../../Atlas/Atlas-Communication" }
+atlas-smr-application = { path = "../../Atlas/Atlas-SMR-Application" }
+atlas-core = { path = "../../Atlas/Atlas-Core" }
+atlas-capnp = { path = "../../Atlas/Atlas-capnp", optional = true }
+atlas-metrics = {path = "../../Atlas/Atlas-Metrics"}
 
 capnp = { version = "0.16" }
 fastrand = "1.7.0"
diff --git a/febft-state-transfer/Cargo.toml b/febft-state-transfer/Cargo.toml
index aca2a5cf..da7df931 100644
--- a/febft-state-transfer/Cargo.toml
+++ b/febft-state-transfer/Cargo.toml
@@ -10,7 +10,8 @@ edition = "2021"
 serialize_serde = ["serde"]
 serialize_capnp = ["atlas-capnp", "capnp"]
 
-default = ["serialize_serde", "atlas-communication/serialize_serde", "atlas-execution/serialize_serde", "atlas-common/serialize_serde", "atlas-core/serialize_serde"]
+default = ["serialize_serde", "atlas-communication/serialize_serde",
+    "atlas-smr-application/serialize_serde", "atlas-common/serialize_serde", "atlas-core/serialize_serde"]
 
 [dependencies]
 
@@ -18,9 +19,9 @@ serde = { version = "*", optional = true }
 capnp = { version = "0.16.1", optional = true }
 log = "0.4.17"
 
-atlas-capnp = { path = "../../Atlas/atlas-capnp", optional = true }
-atlas-execution = { path = "../../Atlas/atlas-execution" }
-atlas-common = { path = "../../Atlas/atlas-common" }
-atlas-communication = { path = "../../Atlas/atlas-communication" }
-atlas-core = { path = "../../Atlas/atlas-core" }
-atlas-metrics = { path = "../../Atlas/atlas-metrics" }
\ No newline at end of file
+atlas-capnp = { path = "../../Atlas/Atlas-capnp", optional = true }
+atlas-smr-application = { path = "../../Atlas/Atlas-SMR-Application" }
+atlas-common = { path = "../../Atlas/Atlas-Common" }
+atlas-communication = { path = "../../Atlas/Atlas-Communication" }
+atlas-core = { path = "../../Atlas/Atlas-Core" }
+atlas-metrics = { path = "../../Atlas/Atlas-Metrics" }
\ No newline at end of file

From cab3edd2105e99680bc4da9fa4f6580923a5c339 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 28 Sep 2023 18:33:12 +0100
Subject: [PATCH 50/80] Fix env var parsing in microbench mark async. Fix RAM
 over allocation in proposer, causing a lot of unused RAM to be allocated and
 thrown into the log, causing ridiculous amounts of virtual memory usage.

---
 febft-pbft-consensus/src/bft/proposer/mod.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index edda4232..09cc30c5 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -272,7 +272,7 @@ impl<D, NT> Proposer<D, NT>
                 //swap in the new vec and take the previous one to the threadpool
                 let new_accumulated_vec =
                     std::mem::replace(&mut propose.currently_accumulated,
-                                      Vec::with_capacity(self.target_global_batch_size * 2));
+                                      vec![]);
 
                 let executor_handle = self.executor_handle.clone();
 
@@ -348,7 +348,7 @@ impl<D, NT> Proposer<D, NT>
                     };
 
                     let current_batch = std::mem::replace(&mut propose.currently_accumulated,
-                                                          next_batch.unwrap_or(Vec::with_capacity(self.max_batch_size * 2)));
+                                                          next_batch.unwrap_or_else(|| vec![]));
 
                     self.propose(seq, &view, current_batch);
 

From cc9bc2fd6bbb3831216e799802ef1bc5456d1dbf Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 28 Sep 2023 18:44:28 +0100
Subject: [PATCH 51/80] Avoid memory allocation all together when it's not
 needed

---
 febft-pbft-consensus/src/bft/proposer/mod.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index 09cc30c5..af099b65 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -272,7 +272,7 @@ impl<D, NT> Proposer<D, NT>
                 //swap in the new vec and take the previous one to the threadpool
                 let new_accumulated_vec =
                     std::mem::replace(&mut propose.currently_accumulated,
-                                      vec![]);
+                                      Vec::new());
 
                 let executor_handle = self.executor_handle.clone();
 
@@ -348,7 +348,7 @@ impl<D, NT> Proposer<D, NT>
                     };
 
                     let current_batch = std::mem::replace(&mut propose.currently_accumulated,
-                                                          next_batch.unwrap_or_else(|| vec![]));
+                                                          next_batch.unwrap_or_else(|| Vec::new()));
 
                     self.propose(seq, &view, current_batch);
 

From 8192b4c8a428d9ca0245105884b0f5d3c8d103a1 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Sun, 1 Oct 2023 21:46:33 +0100
Subject: [PATCH 52/80] More work on the abstraction of the decision log. Still
 undecided on how I'll approach the storage of the last decision.

---
 .../src/bft/consensus/accessory/mod.rs        |  29 +-
 .../bft/consensus/accessory/replica/mod.rs    |  26 +-
 .../src/bft/consensus/decision/mod.rs         | 150 ++++-----
 febft-pbft-consensus/src/bft/consensus/mod.rs |  32 +-
 .../src/bft/log/decided/mod.rs                |  26 ++
 .../src/bft/log/deciding/mod.rs               | 311 ++++++++++++++++++
 febft-pbft-consensus/src/bft/message/mod.rs   |   2 +-
 .../src/bft/message/serialize/mod.rs          |   2 +-
 febft-pbft-consensus/src/bft/mod.rs           |  13 +-
 .../src/bft/msg_log/decided_log/mod.rs        |   4 +-
 .../src/bft/msg_log/decisions/mod.rs          |  16 +-
 febft-pbft-consensus/src/bft/msg_log/mod.rs   |   2 +-
 febft-pbft-consensus/src/bft/proposer/mod.rs  |   6 +-
 .../src/bft/sync/follower_sync/mod.rs         |  11 +-
 febft-pbft-consensus/src/bft/sync/mod.rs      |   6 +-
 .../src/bft/sync/replica_sync/mod.rs          |   8 +-
 16 files changed, 499 insertions(+), 145 deletions(-)
 create mode 100644 febft-pbft-consensus/src/bft/log/decided/mod.rs
 create mode 100644 febft-pbft-consensus/src/bft/log/deciding/mod.rs

diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
index 5514ad7d..ab9594c3 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
@@ -1,13 +1,14 @@
 use std::sync::Arc;
+
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
+use crate::bft::log::deciding::WorkingDecisionLog;
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::msg_log::deciding_log::DecidingLog;
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
-use crate::bft::PBFT;
 use crate::bft::sync::view::ViewInfo;
 
 pub mod replica;
@@ -20,40 +21,40 @@ pub enum ConsensusDecisionAccessory<D>
 
 pub trait AccessoryConsensus<D> where D: ApplicationData + 'static,{
     /// Handle the reception of a pre-prepare message without having completed the pre prepare phase
-    fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle the prepare phase having been completed
-    fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
                                               node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a prepare message processed during the preparing phase without having
     /// reached a quorum
-    fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a prepare message processed during the prepare phase when a quorum
     /// has been achieved
-    fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_preparing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a commit message processed during the preparing phase without having
     /// reached a quorum
-    fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
                                        node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a commit message processed during the prepare phase when a quorum has been reached
-    fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_committing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
                                     node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
@@ -61,7 +62,7 @@ pub trait AccessoryConsensus<D> where D: ApplicationData + 'static,{
 
 impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
     where D: ApplicationData + 'static{
-    fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
@@ -73,7 +74,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         }
     }
 
-    fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                               view: &ViewInfo,
                                               msg: StoredConsensusMessage<D::Request>,
                                               node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static{
@@ -85,7 +86,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         }
     }
 
-    fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
@@ -97,7 +98,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         }
     }
 
-    fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_preparing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>,
                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
@@ -109,7 +110,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         }
     }
 
-    fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>,
                                        node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
@@ -121,7 +122,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         }
     }
 
-    fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_committing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>,
                                     node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index 1036b9c1..76123c52 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -10,12 +10,12 @@ use atlas_common::threadpool;
 use atlas_communication::message::{SerializedMessage, StoredMessage, StoredSerializedProtocolMessage, WireMessage};
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::{PBFT, SysMsg};
 use crate::bft::consensus::accessory::AccessoryConsensus;
+use crate::bft::log::deciding::WorkingDecisionLog;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
-use crate::bft::msg_log::deciding_log::DecidingLog;
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
 use crate::bft::sync::view::ViewInfo;
 
@@ -25,14 +25,14 @@ pub struct ReplicaAccessory<D>
 }
 
 impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
-    where D: ApplicationData + 'static,{
-    fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    where D: ApplicationData + 'static, {
+    fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>,
                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 
     fn handle_pre_prepare_phase_completed<NT>(&mut self,
-                                              deciding_log: &DecidingLog<D::Request>,
+                                              deciding_log: &WorkingDecisionLog<D::Request>,
                                               view: &ViewInfo,
                                               _msg: StoredConsensusMessage<D::Request>,
                                               node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
@@ -51,10 +51,10 @@ impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
 
         threadpool::execute(move || {
             let message = PBFTMessage::Consensus(ConsensusMessage::new(
-                    seq,
-                    view_seq,
-                    ConsensusMessageKind::Commit(current_digest.clone()),
-                ));
+                seq,
+                view_seq,
+                ConsensusMessageKind::Commit(current_digest.clone()),
+            ));
 
             let (message, digest) = node_clone.serialize_digest_message(message).unwrap();
 
@@ -108,11 +108,11 @@ impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
         node.broadcast_signed(message, targets.into_iter());
     }
 
-    fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 
-    fn handle_preparing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_preparing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                    view: &ViewInfo,
                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {
         let node_id = node.id();
@@ -150,11 +150,11 @@ impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
         deciding_log.batch_meta().lock().unwrap().commit_sent_time = Utc::now();
     }
 
-    fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                        view: &ViewInfo,
                                        msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 
-    fn handle_committing_quorum<NT>(&mut self, deciding_log: &DecidingLog<D::Request>,
+    fn handle_committing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                     view: &ViewInfo,
                                     msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 }
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index fa3a2652..8c9bb223 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -15,11 +15,12 @@ use atlas_core::messages::ClientRqInfo;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog};
 use atlas_core::timeouts::Timeouts;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_duration;
 
 use crate::bft::consensus::accessory::{AccessoryConsensus, ConsensusDecisionAccessory};
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
+use crate::bft::log::deciding::WorkingDecisionLog;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{ConsensusMetrics, PRE_PREPARE_ANALYSIS_ID};
@@ -74,24 +75,27 @@ pub enum DecisionPollStatus<O> {
 }
 
 #[derive(Debug, Clone)]
-pub enum DecisionStatus {
+pub enum DecisionStatus<O> {
     /// A particular node tried voting twice.
     VotedTwice(NodeId),
+    // Returned when a node ignores a message
+    MessageIgnored,
+    /// The message has been queued for later execution
+    /// As such this consensus decision should be signaled
+    MessageQueued,
     /// A `febft` quorum still hasn't made a decision
     /// on a client request to be executed.
-    Deciding,
+    Deciding(StoredMessage<ConsensusMessage<O>>),
     /// Transitioned to another next phase of the consensus decision
-    Transitioned,
-    /// The message has been queued for later execution
-    /// As such this consensus decision should be signaled
-    Queued,
+    Transitioned(StoredMessage<ConsensusMessage<O>>),
     /// A `febft` quorum decided on the execution of
     /// the batch of requests with the given digests.
     /// The first digest is the digest of the Prepare message
     /// And therefore the entire batch digest
     /// THe second Vec<Digest> is a vec with digests of the requests contained in the batch
     /// The third is the messages that should be persisted for this batch to be considered persisted
-    Decided,
+    Decided(StoredMessage<ConsensusMessage<O>>),
+    DecidedIgnored
 }
 
 /// A message queue for this particular consensus instance
@@ -111,10 +115,8 @@ pub struct ConsensusDecision<D, PL> where D: ApplicationData + 'static, {
     phase: DecisionPhase,
     /// The queue of messages for this consensus instance
     message_queue: MessageQueue<D::Request>,
-    /// The log of messages for this consensus instance
-    message_log: DecidingLog<D::Request>,
-    /// Persistent log reference
-    persistent_log: PL,
+    /// The working decision log
+    working_log: WorkingDecisionLog<D::Request>,
     /// Accessory to the base consensus state machine
     accessory: ConsensusDecisionAccessory<D>,
     // Metrics about the consensus instance
@@ -180,8 +182,7 @@ impl<D, PL> ConsensusDecision<D, PL>
             seq: seq_no,
             phase: DecisionPhase::Initialize,
             message_queue: MessageQueue::new(),
-            message_log: DecidingLog::new(node_id, seq_no, view),
-            persistent_log,
+            working_log: WorkingDecisionLog::new(node_id, seq_no, view),
             accessory: ConsensusDecisionAccessory::Replica(ReplicaAccessory::new()),
             consensus_metrics: ConsensusMetrics::new(),
         }
@@ -195,8 +196,7 @@ impl<D, PL> ConsensusDecision<D, PL>
             seq: seq_no,
             phase: DecisionPhase::Initialize,
             message_queue,
-            message_log: DecidingLog::new(node_id, seq_no, view),
-            persistent_log,
+            working_log: WorkingDecisionLog::new(node_id, seq_no, view),
             accessory: ConsensusDecisionAccessory::Replica(ReplicaAccessory::new()),
             consensus_metrics: ConsensusMetrics::new(),
         }
@@ -254,7 +254,7 @@ impl<D, PL> ConsensusDecision<D, PL>
 
     /// Update the current view of this consensus instance
     pub fn update_current_view(&mut self, view: &ViewInfo) {
-        self.message_log.update_current_view(view);
+        self.working_log.update_current_view(view);
     }
 
     /// Process a message relating to this consensus instance
@@ -264,7 +264,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                                synchronizer: &Synchronizer<D>,
                                timeouts: &Timeouts,
                                log: &mut Log<D, PL>,
-                               node: &Arc<NT>) -> Result<DecisionStatus>
+                               node: &Arc<NT>) -> Result<DecisionStatus<D::Request>>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
               PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let view = synchronizer.view();
@@ -277,7 +277,7 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                 self.queue(header, message);
 
-                return Ok(DecisionStatus::Queued);
+                return Ok(DecisionStatus::MessageQueued);
             }
             DecisionPhase::PrePreparing(received) => {
                 let received = match message.kind() {
@@ -287,7 +287,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         debug!("{:?} // Dropped {:?} because of view {:?} vs {:?} (ours) header {:?}",
                             self.node_id, message, message.view(), synchronizer.view().sequence_number(), header);
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
                     ConsensusMessageKind::PrePrepare(_)
                     if !view.leader_set().contains(&header.from()) => {
@@ -295,7 +295,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         debug!("{:?} // Dropped {:?} because the sender was not the leader {:?} vs {:?} (ours)",
                         self.node_id,message, header.from(), view.leader());
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
                     ConsensusMessageKind::PrePrepare(_)
                     if message.sequence_number() != self.seq => {
@@ -303,7 +303,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         warn!("{:?} // Dropped {:?} because the sequence number was not the same {:?} vs {:?} (ours)",
                             self.node_id, message, message.sequence_number(), self.seq);
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
                     ConsensusMessageKind::Prepare(d) => {
                         debug!("{:?} // Received {:?} from {:?} while in prepreparing ",
@@ -311,7 +311,7 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                         self.message_queue.queue_prepare(StoredMessage::new(header, message));
 
-                        return Ok(DecisionStatus::Queued);
+                        return Ok(DecisionStatus::MessageQueued);
                     }
                     ConsensusMessageKind::Commit(d) => {
                         debug!("{:?} // Received {:?} from {:?} while in pre preparing",
@@ -319,7 +319,7 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                         self.message_queue.queue_commit(StoredMessage::new(header, message));
 
-                        return Ok(DecisionStatus::Queued);
+                        return Ok(DecisionStatus::MessageQueued);
                     }
                     ConsensusMessageKind::PrePrepare(_) => {
                         // Everything checks out, we can now process the message
@@ -333,22 +333,18 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                 let pre_prepare_received_time = Utc::now();
 
-                let stored_msg = Arc::new(ReadOnly::new(StoredMessage::new(header, message)));
-
+                //TODO: Try out cloning each request on this method,
                 let mut digests = request_batch_received(
-                    &stored_msg,
+                    &message,
                     timeouts,
                     synchronizer,
-                    &mut self.message_log,
+                    &mut self.working_log,
                 );
 
-                let batch_metadata = self.message_log.process_pre_prepare(stored_msg.clone(),
-                                                                          stored_msg.header().digest().clone(),
-                                                                          digests)?;
+                let batch_metadata = self.working_log.process_pre_prepare(header.clone(), &message,
+                                                                          header.digest().clone(), digests)?;
 
-                self.persistent_log.write_message(OperationMode::NonBlockingSync(None), stored_msg.clone())?;
-
-                let mut result = DecisionStatus::Deciding;
+                let mut result ;
 
                 self.phase = if received == view.leader_set().len() {
                     let batch_metadata = batch_metadata.unwrap();
@@ -360,13 +356,13 @@ impl<D, PL> ConsensusDecision<D, PL>
                     //We are now ready to broadcast our prepare message and move to the next phase
                     {
                         //Update batch meta
-                        let mut meta_guard = self.message_log.batch_meta().lock().unwrap();
+                        let mut meta_guard = self.working_log.batch_meta().lock().unwrap();
 
                         meta_guard.prepare_sent_time = Utc::now();
                         meta_guard.pre_prepare_received_time = pre_prepare_received_time;
                     }
 
-                    self.consensus_metrics.all_pre_prepares_recvd(self.message_log.current_batch_size());
+                    self.consensus_metrics.all_pre_prepares_recvd(self.working_log.current_batch_size());
 
                     let current_digest = batch_metadata.batch_digest();
 
@@ -374,13 +370,13 @@ impl<D, PL> ConsensusDecision<D, PL>
                     // The digest of the batch and the order of the batches
                     log.all_batches_received(batch_metadata);
 
-                    self.accessory.handle_pre_prepare_phase_completed(&self.message_log,
+                    self.accessory.handle_pre_prepare_phase_completed(&self.working_log,
                                                                       &view, stored_msg.clone(), node);
 
                     self.message_queue.signal();
 
                     // Mark that we have transitioned to the next phase
-                    result = DecisionStatus::Transitioned;
+                    result = DecisionStatus::Transitioned(StoredMessage::new(header, message));
 
                     // We no longer start the count at 1 since all leaders must also send the prepare
                     // message with the digest of the entire batch
@@ -389,9 +385,11 @@ impl<D, PL> ConsensusDecision<D, PL>
                     debug!("{:?} // Received pre prepare message {:?} from {:?}. Current received {:?}",
                         self.node_id, stored_msg.message(), stored_msg.header().from(), received);
 
-                    self.accessory.handle_partial_pre_prepare(&self.message_log,
+                    self.accessory.handle_partial_pre_prepare(&self.working_log,
                                                               &view, stored_msg.clone(), &**node);
 
+                    result = DecisionStatus::Deciding(StoredMessage::new(header, message));
+
                     DecisionPhase::PrePreparing(received)
                 };
 
@@ -404,7 +402,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         warn!("{:?} // Dropped pre prepare message because we are in the preparing phase",
                             self.node_id);
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
                     ConsensusMessageKind::Commit(d) => {
                         debug!("{:?} // Received {:?} from {:?} while in preparing phase",
@@ -412,28 +410,28 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                         self.message_queue.queue_commit(StoredMessage::new(header, message));
 
-                        return Ok(DecisionStatus::Queued);
+                        return Ok(DecisionStatus::MessageQueued);
                     }
                     ConsensusMessageKind::Prepare(_) if message.view() != view.sequence_number() => {
                         // drop proposed value in a different view (from different leader)
                         warn!("{:?} // Dropped prepare message because of view {:?} vs {:?} (ours)",
                             self.node_id, message.view(), view.sequence_number());
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
                     ConsensusMessageKind::Prepare(_) if message.sequence_number() != self.seq => {
                         // drop proposed value in a different view (from different leader)
                         warn!("{:?} // Dropped prepare message because of seq no {:?} vs {:?} (Ours)",
                             self.node_id, message.sequence_number(), self.seq);
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
-                    ConsensusMessageKind::Prepare(d) if *d != self.message_log.current_digest().unwrap() => {
+                    ConsensusMessageKind::Prepare(d) if *d != self.working_log.current_digest().unwrap() => {
                         // drop msg with different digest from proposed value
                         warn!("{:?} // Dropped prepare message {:?} from {:?} because of digest {:?} vs {:?} (ours)",
-                            self.node_id, message.sequence_number(), header.from(), d, self.message_log.current_digest());
+                            self.node_id, message.sequence_number(), header.from(), d, self.working_log.current_digest());
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
                     ConsensusMessageKind::Prepare(_) => {
                         // Everything checks out, we can now process the message
@@ -445,38 +443,36 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.consensus_metrics.first_prepare_recvd();
                 }
 
-                let stored_msg = Arc::new(ReadOnly::new(StoredMessage::new(header, message)));
-
-                self.message_log.process_message(stored_msg.clone())?;
+                self.working_log.process_message(&header, &message)?;
 
-                self.persistent_log.write_message(OperationMode::NonBlockingSync(None), stored_msg.clone())?;
-
-                let mut result = DecisionStatus::Deciding;
+                let result;
 
                 self.phase = if received == view.params().quorum() {
                     info!("{:?} // Completed prepare phase with all prepares Seq {:?} with prepare from {:?}", node.id(), self.sequence_number(), header.from());
 
-                    self.message_log.batch_meta().lock().unwrap().commit_sent_time = Utc::now();
+                    self.working_log.batch_meta().lock().unwrap().commit_sent_time = Utc::now();
                     self.consensus_metrics.prepare_quorum_recvd();
 
                     let seq_no = self.sequence_number();
-                    let current_digest = self.message_log.current_digest().unwrap();
+                    let current_digest = self.working_log.current_digest().unwrap();
 
-                    self.accessory.handle_preparing_quorum(&self.message_log, &view,
+                    self.accessory.handle_preparing_quorum(&self.working_log, &view,
                                                            stored_msg.clone(), &**node);
 
                     self.message_queue.signal();
 
-                    result = DecisionStatus::Transitioned;
+                    result = DecisionStatus::Transitioned(StoredMessage::new(header, message));
 
                     DecisionPhase::Committing(0)
                 } else {
                     debug!("{:?} // Received prepare message {:?} from {:?}. Current count {}",
                         self.node_id, stored_msg.message().sequence_number(), header.from(), received);
 
-                    self.accessory.handle_preparing_no_quorum(&self.message_log, &view,
+                    self.accessory.handle_preparing_no_quorum(&self.working_log, &view,
                                                               stored_msg.clone(), &**node);
 
+                    result = DecisionStatus::Deciding(StoredMessage::new(header, message));
+
                     DecisionPhase::Preparing(received)
                 };
 
@@ -489,21 +485,21 @@ impl<D, PL> ConsensusDecision<D, PL>
                         warn!("{:?} // Dropped commit message because of seq no {:?} vs {:?} (Ours)",
                             self.node_id, message.sequence_number(), self.seq);
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
                     ConsensusMessageKind::Commit(_) if message.view() != view.sequence_number() => {
                         // drop proposed value in a different view (from different leader)
                         warn!("{:?} // Dropped commit message because of view {:?} vs {:?} (ours)",
                             self.node_id, message.view(), view.sequence_number());
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
-                    ConsensusMessageKind::Commit(d) if *d != self.message_log.current_digest().unwrap() => {
+                    ConsensusMessageKind::Commit(d) if *d != self.working_log.current_digest().unwrap() => {
                         // drop msg with different digest from proposed value
                         warn!("{:?} // Dropped commit message {:?} from {:?} because of digest {:?} vs {:?} (ours)",
                             self.node_id, message.sequence_number(), header.from(), d, self.message_log.current_digest());
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
                     ConsensusMessageKind::Commit(_) => {
                         received + 1
@@ -511,7 +507,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     _ => {
                         // Any message relating to any other phase other than commit is not accepted
 
-                        return Ok(DecisionStatus::Deciding);
+                        return Ok(DecisionStatus::MessageIgnored);
                     }
                 };
 
@@ -519,11 +515,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.consensus_metrics.first_commit_recvd();
                 }
 
-                let stored_msg = Arc::new(ReadOnly::new(StoredMessage::new(header, message)));
-
-                self.message_log.process_message(stored_msg.clone())?;
-
-                self.persistent_log.write_message(OperationMode::NonBlockingSync(None), stored_msg.clone())?;
+                self.working_log.process_message(&header, &message);
 
                 return if received == view.params().quorum() {
                     info!("{:?} // Completed commit phase with all commits Seq {:?} with commit from {:?}", node.id(), self.sequence_number(),
@@ -531,29 +523,29 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                     self.phase = DecisionPhase::Decided;
 
-                    self.message_log.batch_meta().lock().unwrap().consensus_decision_time = Utc::now();
+                    self.working_log.batch_meta().lock().unwrap().consensus_decision_time = Utc::now();
 
                     self.consensus_metrics.commit_quorum_recvd();
 
-                    self.accessory.handle_committing_quorum(&self.message_log, &view,
+                    self.accessory.handle_committing_quorum(&self.working_log, &view,
                                                             stored_msg.clone(), &**node);
 
-                    Ok(DecisionStatus::Decided)
+                    Ok(DecisionStatus::Decided(StoredMessage::new(header, message)))
                 } else {
                     debug!("{:?} // Received commit message {:?} from {:?}. Current count {}",
                         self.node_id, stored_msg.message().sequence_number(), header.from(), received);
 
                     self.phase = DecisionPhase::Committing(received);
 
-                    self.accessory.handle_committing_no_quorum(&self.message_log, &view,
+                    self.accessory.handle_committing_no_quorum(&self.working_log, &view,
                                                                stored_msg.clone(), &**node);
 
-                    Ok(DecisionStatus::Deciding)
+                    Ok(DecisionStatus::Deciding(StoredMessage::new(header, message)))
                 };
             }
             DecisionPhase::Decided => {
                 //Drop unneeded messages
-                Ok(DecisionStatus::Decided)
+                Ok(DecisionStatus::DecidedIgnored)
             }
         };
     }
@@ -570,7 +562,7 @@ impl<D, PL> ConsensusDecision<D, PL>
     /// Finalize this consensus decision and return the information about the batch
     pub fn finalize(self) -> Result<CompletedBatch<D::Request>> {
         if let DecisionPhase::Decided = self.phase {
-            self.message_log.finish_processing_batch()
+            self.working_log.finish_processing_batch()
                 .ok_or(Error::simple_with_msg(ErrorKind::Consensus, "Failed to finalize batch"))
         } else {
             Err(Error::simple_with_msg(ErrorKind::Consensus, "Cannot finalize batch that is not decided"))
@@ -578,11 +570,7 @@ impl<D, PL> ConsensusDecision<D, PL>
     }
 
     pub fn deciding(&self, f: usize) -> IncompleteProof {
-        self.message_log.deciding(f)
-    }
-
-    pub fn message_log(&self) -> &DecidingLog<D::Request> {
-        &self.message_log
+        self.working_log.deciding(f)
     }
 
     pub fn phase(&self) -> &DecisionPhase {
@@ -599,10 +587,10 @@ impl<D, PL> Orderable for ConsensusDecision<D, PL>
 
 #[inline]
 fn request_batch_received<D>(
-    pre_prepare: &StoredConsensusMessage<D::Request>,
+    pre_prepare: &ConsensusMessage<D::Request>,
     timeouts: &Timeouts,
     synchronizer: &Synchronizer<D>,
-    log: &DecidingLog<D::Request>,
+    log: &WorkingDecisionLog<D::Request>,
 ) -> Vec<ClientRqInfo>
     where
         D: ApplicationData + 'static,
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 66008ff9..135a7d83 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -17,8 +17,8 @@ use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::timeouts::Timeouts;
-use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::ExecutorHandle;
+use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_increment;
 
 use crate::bft::{PBFT, SysMsg};
@@ -40,6 +40,10 @@ pub mod accessory;
 pub enum ConsensusStatus {
     /// A particular node tried voting twice.
     VotedTwice(NodeId),
+    /// The message has been ignored
+    MessageIgnored,
+    /// The message has been queued
+    MessageQueued,
     /// A `febft` quorum still hasn't made a decision
     /// on a client request to be executed.
     Deciding,
@@ -376,7 +380,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             Either::Right(i) if i > 0 => {
                 self.enqueue_other_view_message(i, header, message);
 
-                return Ok(ConsensusStatus::Deciding);
+                return Ok(ConsensusStatus::MessageQueued);
             }
             Either::Right(_) => {}
             Either::Left(_) => {
@@ -384,7 +388,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
                 debug!("{:?} // Ignoring consensus message {:?} received from {:?} as we are already in view {:?}",
                     self.node_id, message, header.from(), self.curr_view.sequence_number());
 
-                return Ok(ConsensusStatus::Deciding);
+                return Ok(ConsensusStatus::MessageIgnored);
             }
         };
 
@@ -393,7 +397,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             Either::Left(_) => {
                 debug!("Message {:?} from {:?} is behind our current sequence no {:?}. Ignoring", message, header.from(), self.seq_no, );
 
-                return Ok(ConsensusStatus::Deciding);
+                return Ok(ConsensusStatus::MessageIgnored);
             }
         };
 
@@ -404,7 +408,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
 
             self.tbo_queue.queue(header, message);
 
-            return Ok(ConsensusStatus::Deciding);
+            return Ok(ConsensusStatus::MessageQueued);
         }
 
         // Get the correct consensus instance for this message
@@ -416,19 +420,29 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             DecisionStatus::VotedTwice(node) => {
                 ConsensusStatus::VotedTwice(node)
             }
-            DecisionStatus::Deciding => {
+            DecisionStatus::Deciding(message) => {
                 ConsensusStatus::Deciding
             }
-            DecisionStatus::Queued | DecisionStatus::Transitioned => {
+            DecisionStatus::MessageQueued => {
+                //When we transition phases, we may discover new messages
+                // That were in the queue, so we must be signalled again
+                self.signalled.push_signalled(message_seq);
+
+                ConsensusStatus::MessageQueued
+            }
+            DecisionStatus::Transitioned(message) => {
                 //When we transition phases, we may discover new messages
                 // That were in the queue, so we must be signalled again
                 self.signalled.push_signalled(message_seq);
 
                 ConsensusStatus::Deciding
             }
-            DecisionStatus::Decided => {
+            DecisionStatus::Decided(message) => {
                 ConsensusStatus::Decided
             }
+            DecisionStatus::MessageIgnored => {
+                ConsensusStatus::MessageIgnored
+            }
         })
     }
 
diff --git a/febft-pbft-consensus/src/bft/log/decided/mod.rs b/febft-pbft-consensus/src/bft/log/decided/mod.rs
new file mode 100644
index 00000000..5e9aeb75
--- /dev/null
+++ b/febft-pbft-consensus/src/bft/log/decided/mod.rs
@@ -0,0 +1,26 @@
+use crate::bft::msg_log::decisions::Proof;
+
+/// A necessary decision log for the ability to perform view changes.
+/// Only stores the latest performed decision
+pub struct OngoingDecisionLog<O> {
+    /// The last decision that was performed by the ordering protocol
+    last_decision: Option<Proof<O>>
+}
+
+impl<O> OngoingDecisionLog<O> {
+
+    fn init(last_proof: Option<Proof<O>>) -> Self {
+        OngoingDecisionLog {
+            last_decision: last_proof,
+        }
+    }
+
+    fn install_proof(&mut self, proof: Proof<O>) {
+        self.last_decision = Some(proof)
+    }
+    
+    fn last_decision(&self) -> Option<Proof<O>>  {
+        self.last_decision.clone()
+    }
+    
+}
\ No newline at end of file
diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
new file mode 100644
index 00000000..9104f6a7
--- /dev/null
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -0,0 +1,311 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::iter;
+use std::sync::{Arc, Mutex};
+use std::time::Instant;
+use atlas_common::crypto::hash::{Context, Digest};
+use atlas_common::node_id::NodeId;
+use atlas_common::ordering::{Orderable, SeqNo};
+use atlas_common::error::*;
+use atlas_communication::message::Header;
+use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
+use atlas_metrics::benchmarks::BatchMeta;
+use atlas_metrics::metrics::metric_duration;
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
+use crate::bft::metric::PRE_PREPARE_LOG_ANALYSIS_ID;
+use crate::bft::msg_log::deciding_log::{DecidingLog, FullBatch};
+use crate::bft::msg_log::decisions::{IncompleteProof, ProofMetadata};
+use crate::bft::sync::view::ViewInfo;
+
+pub struct CompletedBatch<O> {
+    seq: SeqNo,
+    // The overall digest of the entire batch
+    digest: Digest,
+    // The ordering of the pre prepare requests
+    pre_prepare_ordering: Vec<Digest>,
+    // The information of the client requests that are contained in this batch
+    client_request_info: Vec<ClientRqInfo>,
+    // The client requests contained in this batch
+    client_requests: Vec<StoredRequestMessage<O>>,
+    batch_meta: BatchMeta,
+}
+
+pub struct WorkingDecisionLog<O> {
+    node_id: NodeId,
+    seq_no: SeqNo,
+
+    duplicate_detection: DuplicateReplicaEvaluator,
+    //The digest of the entire batch that is currently being processed
+    // This will only be calculated when we receive all of the requests necessary
+    // As this digest requires the knowledge of all of them
+    batch_digest: Option<Digest>,
+    // The digests of all received pre prepares
+    // We store a vec of options because we want to store the digests in the correct ordering
+    pre_prepare_digests: Vec<Option<Digest>>,
+    // How many pre prepares have we received
+    current_received_pre_prepares: usize,
+    //The size of batch that is currently being processed. Increases as we receive more pre prepares
+    current_batch_size: usize,
+    //The client requests that are currently being processed
+    //Does not have to follow the correct order, only has to contain the requests
+    client_rqs: Vec<ClientRqInfo>,
+    // The set of leaders that is currently in vigour for this consensus decision
+    leader_set: Vec<NodeId>,
+    // Which hash space should each leader be responsible for
+    request_space_slices: BTreeMap<NodeId, (Vec<u8>, Vec<u8>)>,
+    // Some logging information about metadata
+    batch_meta: Arc<Mutex<BatchMeta>>,
+    // The contained requests per each of the received pre prepares
+    contained_requests: Vec<Option<Vec<StoredRequestMessage<O>>>>,
+}
+
+/// Checks to make sure replicas aren't providing more than one vote for the
+/// Same consensus decision
+#[derive(Default)]
+pub struct DuplicateReplicaEvaluator {
+    // The set of leaders that is currently in vigour for this consensus decision
+    leader_set: Vec<NodeId>,
+    // The set of leaders that have already sent a pre prepare message
+    received_pre_prepare_messages: BTreeSet<NodeId>,
+    // The set of leaders that have already sent a prepare message
+    received_prepare_messages: BTreeSet<NodeId>,
+    // The set of leaders that have already sent a commit message
+    received_commit_messages: BTreeSet<NodeId>,
+}
+
+impl<O> WorkingDecisionLog<O> where O: Clone {
+    pub fn new(node: NodeId, seq: SeqNo, view: &ViewInfo) -> Self {
+        let leader_count = view.leader_set().len();
+        Self {
+            node_id,
+            seq_no: seq,
+            duplicate_detection: Default::default(),
+            batch_digest: None,
+            pre_prepare_digests: iter::repeat(None).take(leader_count).collect(),
+            current_received_pre_prepares: 0,
+            current_batch_size: 0,
+            client_rqs: vec![],
+            leader_set: view.leader_set().clone(),
+            request_space_slices: view.hash_space_division().clone(),
+            batch_meta: Arc::new(Mutex::new(BatchMeta::new())),
+            contained_requests: iter::repeat(None).take(leader_count).collect(),
+        }
+    }
+
+    pub fn update_current_view(&mut self, view: &ViewInfo) {
+        self.leader_set = view.leader_set().clone();
+        self.request_space_slices = view.hash_space_division().clone();
+    }
+
+    pub fn process_pre_prepare(&mut self,
+                               header: Header,
+                               message: &ConsensusMessage<O>,
+                               digest: Digest,
+                               mut batch_rq_digests: Vec<ClientRqInfo>) -> Result<Option<FullBatch>> {
+        let start = Instant::now();
+
+        let sending_leader = header.from();
+
+        let slice = self.request_space_slices.get(&sending_leader)
+            .ok_or(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
+                                          format!("Failed to get request space for leader {:?}. Len: {:?}. {:?}",
+                                                  sending_leader,
+                                                  self.request_space_slices.len(),
+                                                  self.request_space_slices).as_str()))?;
+
+        if sending_leader != self.node_id {
+            // Only check batches from other leaders since we implicitly trust in ourselves
+            for request in &batch_rq_digests {
+                if !crate::bft::sync::view::is_request_in_hash_space(&request.digest(), slice) {
+                    return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
+                                                      "This batch contains requests that are not in the hash space of the leader."));
+                }
+            }
+        }
+
+        self.duplicate_detection.insert_pre_prepare_received(sending_leader)?;
+
+        // Get the correct index for this batch
+        let leader_index = pre_prepare_index_of(&self.leader_set, &sending_leader)?;
+
+        if leader_index >= self.pre_prepare_digests.len() {
+            unreachable!("Cannot insert a pre prepare message that was sent by a leader that is out of bounds")
+        }
+
+        self.pre_prepare_digests[leader_index] = Some(digest);
+        self.contained_requests[leader_index] = Some(match message.kind() {
+            ConsensusMessageKind::PrePrepare(requests) => {
+                requests.clone()
+            }
+            _ => unreachable!()
+        });
+
+        self.current_received_pre_prepares += 1;
+
+        self.current_batch_size += batch_rq_digests.len();
+
+        self.client_rqs.append(&mut batch_rq_digests);
+
+        metric_duration(PRE_PREPARE_LOG_ANALYSIS_ID, start.elapsed());
+
+        // if we have received all of the messages in the set, calculate the digest.
+        Ok(if self.current_received_pre_prepares == self.leader_set.len() {
+            // We have received all of the required batches
+            let result = self.calculate_instance_digest();
+
+            let (digest, ordering) = result
+                .ok_or(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog, "Failed to calculate instance digest"))?;
+
+            self.batch_digest = Some(digest.clone());
+
+            Some(ProofMetadata::new(self.seq_no, digest, ordering))
+        } else {
+            None
+        })
+    }
+
+    /// Process the message received
+    pub(crate) fn process_message<O>(&mut self, header: &Header, message: &ConsensusMessage<O>) -> Result<()> {
+        match message.message().kind() {
+            ConsensusMessageKind::Prepare(_) => {
+                self.duplicate_detection.insert_prepare_received(message.header().from())?;
+            }
+            ConsensusMessageKind::Commit(_) => {
+                self.duplicate_detection.insert_commit_received(message.header().from())?;
+            }
+            _ => unreachable!()
+        }
+
+        Ok(())
+    }
+
+    /// Getter for batch_meta
+    pub fn batch_meta(&self) -> &Arc<Mutex<BatchMeta>> {
+        &self.batch_meta
+    }
+
+    /// Getter for the current digest
+    pub fn current_digest(&self) -> Option<Digest> {
+        self.batch_digest.clone()
+    }
+
+    /// Get the current batch size in this consensus decision
+    pub fn current_batch_size(&self) -> usize { self.current_batch_size }
+
+    /// Calculate the instance of a completed consensus pre prepare phase with
+    /// all the batches received
+    fn calculate_instance_digest(&self) -> Option<(Digest, Vec<Digest>)> {
+        let mut ctx = Context::new();
+
+        let mut batch_ordered_digests = Vec::with_capacity(self.pre_prepare_digests.len());
+
+        for order_digest in &self.pre_prepare_digests {
+            if let Some(digest) = order_digest.clone() {
+                ctx.update(digest.as_ref());
+                batch_ordered_digests.push(digest);
+            } else {
+                return None;
+            }
+        }
+
+        Some((ctx.finish(), batch_ordered_digests))
+    }
+
+
+    /// Get the current decision
+    pub fn deciding(&self, f: usize) -> IncompleteProof {
+        todo!()
+    }
+
+    /// Indicate that the batch is finished processing and
+    /// return the relevant information for it
+    pub fn finish_processing_batch(self) -> Option<CompletedBatch<O>> {
+        let new_meta = BatchMeta::new();
+        let batch_meta = std::mem::replace(&mut *self.batch_meta().lock().unwrap(), new_meta);
+
+        let current_digest = self.batch_digest?;
+
+        let pre_prepare_ordering = self.pre_prepare_digests.into_iter().map(|elem| elem.unwrap()).collect();
+
+        let mut requests = Vec::with_capacity(self.current_batch_size);
+
+        for pre_prepare_request in self.contained_requests {
+            requests.append(&mut pre_prepare_request.unwrap());
+        }
+
+        Some(CompletedBatch {
+            seq: self.seq_no,
+            digest: current_digest,
+            pre_prepare_ordering,
+            client_request_info: self.client_rqs,
+            batch_meta,
+            client_requests: requests
+        })
+    }
+}
+
+impl Orderable for WorkingDecisionLog {
+    fn sequence_number(&self) -> SeqNo {
+        self.seq_no
+    }
+}
+
+impl DuplicateReplicaEvaluator {
+    fn insert_pre_prepare_received(&mut self, node_id: NodeId) -> Result<()> {
+        if !self.received_pre_prepare_messages.insert(node_id) {
+            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
+                                              "We have already received a message from that leader."));
+        }
+
+        Ok(())
+    }
+    fn insert_prepare_received(&mut self, node_id: NodeId) -> Result<()> {
+        if !self.received_prepare_messages.insert(node_id) {
+            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
+                                              "We have already received a message from that leader."));
+        }
+
+        Ok(())
+    }
+    fn insert_commit_received(&mut self, node_id: NodeId) -> Result<()> {
+        if !self.received_commit_messages.insert(node_id) {
+            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
+                                              "We have already received a message from that leader."));
+        }
+
+        Ok(())
+    }
+}
+
+
+pub fn pre_prepare_index_from_digest_opt(prepare_set: &Vec<Option<Digest>>, digest: &Digest) -> Result<usize> {
+    match prepare_set.iter().position(|pre_prepare| pre_prepare.map(|d| d == *digest).unwrap_or(false)) {
+        None => {
+            Err(Error::simple_with_msg(ErrorKind::Consensus, "Pre prepare is not part of the pre prepare set"))
+        }
+        Some(pos) => {
+            Ok(pos)
+        }
+    }
+}
+
+pub fn pre_prepare_index_of_from_digest(prepare_set: &Vec<Digest>, preprepare: &Digest) -> Result<usize> {
+    match prepare_set.iter().position(|pre_prepare| *pre_prepare == *preprepare) {
+        None => {
+            Err(Error::simple_with_msg(ErrorKind::Consensus, "Pre prepare is not part of the pre prepare set"))
+        }
+        Some(pos) => {
+            Ok(pos)
+        }
+    }
+}
+
+pub fn pre_prepare_index_of(leader_set: &Vec<NodeId>, proposer: &NodeId) -> Result<usize> {
+    match leader_set.iter().position(|node| *node == *proposer) {
+        None => {
+            Err(Error::simple_with_msg(ErrorKind::Consensus, "Proposer is not part of the leader set"))
+        }
+        Some(pos) => {
+            Ok(pos)
+        }
+    }
+}
diff --git a/febft-pbft-consensus/src/bft/message/mod.rs b/febft-pbft-consensus/src/bft/message/mod.rs
index 86d6a6f7..f0b12d80 100644
--- a/febft-pbft-consensus/src/bft/message/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/mod.rs
@@ -17,7 +17,7 @@ use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::Header;
 use atlas_core::messages::{RequestMessage, StoredRequestMessage};
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::msg_log::decisions::CollectData;
 use crate::bft::sync::LeaderCollects;
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index fcdfe0ba..f1f48c50 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -24,7 +24,7 @@ use atlas_core::messages::SystemMessage;
 use atlas_core::ordering_protocol::networking::serialize::{OrderingProtocolMessage, PermissionedOrderingProtocolMessage, StatefulOrderProtocolMessage};
 use atlas_core::ordering_protocol::networking::signature_ver::OrderProtocolSignatureVerificationHelper;
 use atlas_core::persistent_log::PersistableOrderProtocol;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata};
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index f5b85bfb..c1f19f80 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -19,18 +19,17 @@ use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::serialize::Serializable;
 use atlas_core::messages::Protocol;
-use atlas_core::ordering_protocol::{LoggableMessage, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, PermissionedOrderingProtocol, ProtocolConsensusDecision, SerProof, SerProofMetadata, View};
+use atlas_core::ordering_protocol::{ OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, PermissionedOrderingProtocol, ProtocolConsensusDecision, SerProof, SerProofMetadata, View};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::networking::serialize::{NetworkView, OrderingProtocolMessage};
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::{ReconfigurableOrderProtocol, ReconfigurationAttemptResult};
-use atlas_core::ordering_protocol::stateful_order_protocol::{DecLog, StatefulOrderProtocol};
 use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol, StatefulOrderingProtocolLog};
 use atlas_core::reconfiguration_protocol::ReconfigurationProtocol;
 use atlas_core::request_pre_processing::RequestPreProcessor;
 use atlas_core::serialize::ReconfigurationProtocolMessage;
 use atlas_core::timeouts::{RqTimeout, Timeouts};
-use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::ExecutorHandle;
+use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_duration;
 
 use crate::bft::config::PBFTConfig;
@@ -48,6 +47,10 @@ pub mod consensus;
 pub mod proposer;
 pub mod sync;
 pub mod msg_log;
+pub mod log {
+    pub mod deciding;
+    pub mod decided;
+}
 pub mod config;
 pub mod message;
 pub mod observer;
@@ -296,7 +299,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
 
         let OrderingProtocolArgs(executor, timeouts,
                                  pre_processor, batch_input,
-                                 node, persistent_log, quorum) = args;
+                                 node,  quorum) = args;
 
         let sync = Synchronizer::initialize_with_quorum(node_id, view.sequence_number(), quorum.clone(), timeout_dur)?;
 
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index 5fc88739..50bb3224 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -5,8 +5,8 @@ use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_core::ordering_protocol::{DecisionInformation, ProtocolConsensusDecision};
 use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_execution::app::UpdateBatch;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::app::UpdateBatch;
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::message::ConsensusMessageKind;
 use crate::bft::message::serialize::PBFTConsensus;
diff --git a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
index b9f04fb9..4681306c 100644
--- a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
@@ -380,7 +380,21 @@ impl<O> OrderProtocolLog for DecisionLog<O> {
     }
 }
 
-impl<O> OrderProtocolProof for Proof<O> {}
+impl<O> OrderProtocolProof for Proof<O> {
+    fn contained_messages(&self) -> usize {
+        let mut messages = 0;
+
+        for pre_prepare in &self.pre_prepares {
+            //Mark the requests contained in this message for removal
+            messages += match pre_prepare.message().kind() {
+                ConsensusMessageKind::PrePrepare(messages) => messages.len(),
+                _ => 0,
+            };
+        }
+
+        messages
+    }
+}
 
 impl<O> Clone for Proof<O> {
     fn clone(&self) -> Self {
diff --git a/febft-pbft-consensus/src/bft/msg_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/mod.rs
index 529849b1..c6dd978a 100644
--- a/febft-pbft-consensus/src/bft/msg_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/mod.rs
@@ -5,7 +5,7 @@ use atlas_common::node_id::NodeId;
 use atlas_common::ordering::SeqNo;
 use atlas_communication::message::Header;
 use atlas_core::messages::RequestMessage;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::msg_log::decided_log::Log;
 use crate::bft::msg_log::decisions::DecisionLog;
diff --git a/febft-pbft-consensus/src/bft/proposer/mod.rs b/febft-pbft-consensus/src/bft/proposer/mod.rs
index af099b65..b5804913 100644
--- a/febft-pbft-consensus/src/bft/proposer/mod.rs
+++ b/febft-pbft-consensus/src/bft/proposer/mod.rs
@@ -14,9 +14,9 @@ use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::request_pre_processing::{BatchOutput, PreProcessorOutputMessage};
 use atlas_core::timeouts::Timeouts;
-use atlas_execution::app::UnorderedBatch;
-use atlas_execution::ExecutorHandle;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::app::UnorderedBatch;
+use atlas_smr_application::ExecutorHandle;
+use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::{metric_duration, metric_increment, metric_store_count};
 
 use crate::bft::config::ProposerConfig;
diff --git a/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
index 49b535a0..b68d2b9a 100644
--- a/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
@@ -1,17 +1,16 @@
 use std::{marker::PhantomData};
 use atlas_common::ordering::Orderable;
 use atlas_core::messages::ClientRqInfo;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::message::{ConsensusMessageKind};
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
 
 pub struct FollowerSynchronizer<D: ApplicationData> {
-    _phantom: PhantomData<D>
+    _phantom: PhantomData<D>,
 }
 
 impl<D: ApplicationData + 'static> FollowerSynchronizer<D> {
-
     pub fn new() -> Self {
         Self { _phantom: Default::default() }
     }
@@ -23,10 +22,9 @@ impl<D: ApplicationData + 'static> FollowerSynchronizer<D> {
         &self,
         pre_prepare: &StoredConsensusMessage<D::Request>,
     ) -> Vec<ClientRqInfo> {
-
         let requests = match pre_prepare.message().kind() {
-            ConsensusMessageKind::PrePrepare(req) => {req},
-            _ => {panic!()}
+            ConsensusMessageKind::PrePrepare(req) => { req }
+            _ => { panic!() }
         };
 
         let mut digests = Vec::with_capacity(requests.len());
@@ -59,5 +57,4 @@ impl<D: ApplicationData + 'static> FollowerSynchronizer<D> {
 
         digests
     }
-
 }
\ No newline at end of file
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 79d20d5f..b6cfb98a 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -29,10 +29,10 @@ use atlas_core::ordering_protocol::reconfigurable_order_protocol::Reconfiguratio
 use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
 use atlas_core::request_pre_processing::RequestPreProcessor;
 use atlas_core::timeouts::{RqTimeout, Timeouts};
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::consensus::Consensus;
-use crate::bft::message::{ConsensusMessageKind, FwdConsensusMessage, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, FwdConsensusMessage, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::msg_log::decided_log::Log;
 use crate::bft::msg_log::decisions::{CollectData, Proof, StoredConsensusMessage, ViewDecisionPair};
@@ -1681,7 +1681,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     /// proposed, they won't timeout
     pub fn request_batch_received(
         &self,
-        pre_prepare: &StoredConsensusMessage<D::Request>,
+        pre_prepare: &ConsensusMessage<D::Request>,
         timeouts: &Timeouts,
     ) -> Vec<ClientRqInfo> {
         match &self.accessory {
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 23370566..54d6e869 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -18,11 +18,11 @@ use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::persistent_log::OrderingProtocolLog;
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, TimeoutPhase, Timeouts};
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::{metric_duration, metric_increment};
 
 use crate::bft::consensus::Consensus;
-use crate::bft::message::{ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{SYNC_BATCH_RECEIVED_ID, SYNC_STOPPED_COUNT_ID, SYNC_STOPPED_REQUESTS_ID, SYNC_WATCH_REQUESTS_ID};
 use crate::bft::msg_log::decided_log::Log;
@@ -178,12 +178,12 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     /// proposed, they won't timeout
     pub fn received_request_batch(
         &self,
-        pre_prepare: &StoredConsensusMessage<D::Request>,
+        pre_prepare: &ConsensusMessage<D::Request>,
         timeouts: &Timeouts,
     ) -> Vec<ClientRqInfo> {
         let start_time = Instant::now();
 
-        let requests = match pre_prepare.message().kind() {
+        let requests = match pre_prepare.kind() {
             ConsensusMessageKind::PrePrepare(req) => { req }
             _ => {
                 error!("Cannot receive a request that is not a PrePrepare");

From d25f5a5c05a657f37daa8c8e58b33c31bb46158b Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 2 Oct 2023 13:42:26 +0100
Subject: [PATCH 53/80] More work on delivering updates to the decision log

---
 .../src/bft/consensus/accessory/mod.rs        | 26 ++++++++++---------
 .../bft/consensus/accessory/replica/mod.rs    | 15 ++++++-----
 .../src/bft/consensus/decision/mod.rs         | 23 ++++++++--------
 febft-pbft-consensus/src/bft/consensus/mod.rs |  3 ++-
 .../src/bft/log/deciding/mod.rs               |  5 +++-
 5 files changed, 39 insertions(+), 33 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
index ab9594c3..cc1aa6bc 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
@@ -1,4 +1,5 @@
 use std::sync::Arc;
+use atlas_communication::message::Header;
 
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
@@ -6,6 +7,7 @@ use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
 use crate::bft::log::deciding::WorkingDecisionLog;
+use crate::bft::message::ConsensusMessage;
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::msg_log::deciding_log::DecidingLog;
 use crate::bft::msg_log::decisions::StoredConsensusMessage;
@@ -23,40 +25,40 @@ pub trait AccessoryConsensus<D> where D: ApplicationData + 'static,{
     /// Handle the reception of a pre-prepare message without having completed the pre prepare phase
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>,
+                                      header: &Header, msg: &ConsensusMessage<D::Request>,
                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle the prepare phase having been completed
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                               view: &ViewInfo,
-                                              msg: StoredConsensusMessage<D::Request>,
+                                              header: &Header, msg: &ConsensusMessage<D::Request>,
                                               node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a prepare message processed during the preparing phase without having
     /// reached a quorum
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>,
+                                      header: &Header, msg: &ConsensusMessage<D::Request>,
                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a prepare message processed during the prepare phase when a quorum
     /// has been achieved
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                    view: &ViewInfo,
-                                   msg: StoredConsensusMessage<D::Request>,
+                                   header: &Header, msg: &ConsensusMessage<D::Request>,
                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a commit message processed during the preparing phase without having
     /// reached a quorum
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                        view: &ViewInfo,
-                                       msg: StoredConsensusMessage<D::Request>,
+                                       header: &Header, msg: &ConsensusMessage<D::Request>,
                                        node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 
     /// Handle a commit message processed during the prepare phase when a quorum has been reached
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                     view: &ViewInfo,
-                                    msg: StoredConsensusMessage<D::Request>,
+                                    header: &Header, msg: &ConsensusMessage<D::Request>,
                                     node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static;
 }
 
@@ -64,7 +66,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
     where D: ApplicationData + 'static{
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>,
+                                      header: &Header, msg: &ConsensusMessage<D::Request>,
                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
@@ -76,7 +78,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
 
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                               view: &ViewInfo,
-                                              msg: StoredConsensusMessage<D::Request>,
+                                              header: &Header, msg: &ConsensusMessage<D::Request>,
                                               node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static{
         match self {
             ConsensusDecisionAccessory::Follower => {}
@@ -88,7 +90,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>,
+                                      header: &Header, msg: &ConsensusMessage<D::Request>,
                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
@@ -100,7 +102,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
 
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                    view: &ViewInfo,
-                                   msg: StoredConsensusMessage<D::Request>,
+                                   header: &Header, msg: &ConsensusMessage<D::Request>,
                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
@@ -112,7 +114,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
 
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                        view: &ViewInfo,
-                                       msg: StoredConsensusMessage<D::Request>,
+                                       header: &Header, msg: &ConsensusMessage<D::Request>,
                                        node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
@@ -124,7 +126,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
 
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                     view: &ViewInfo,
-                                    msg: StoredConsensusMessage<D::Request>,
+                                    header: &Header, msg: &ConsensusMessage<D::Request>,
                                     node: &NT) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index 76123c52..a25f58d7 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -7,7 +7,7 @@ use log::debug;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::threadpool;
-use atlas_communication::message::{SerializedMessage, StoredMessage, StoredSerializedProtocolMessage, WireMessage};
+use atlas_communication::message::{Header, SerializedMessage, StoredMessage, StoredSerializedProtocolMessage, WireMessage};
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_smr_application::serialize::ApplicationData;
@@ -28,13 +28,13 @@ impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
     where D: ApplicationData + 'static, {
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>,
+                                      header: &Header, msg: &ConsensusMessage<D::Request>,
                                       node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 
     fn handle_pre_prepare_phase_completed<NT>(&mut self,
                                               deciding_log: &WorkingDecisionLog<D::Request>,
                                               view: &ViewInfo,
-                                              _msg: StoredConsensusMessage<D::Request>,
+                                              header: &Header, msg: &ConsensusMessage<D::Request>,
                                               node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
         let my_id = node.id();
         let view_seq = view.sequence_number();
@@ -110,11 +110,11 @@ impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
-                                      msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
+                                      header: &Header, msg: &ConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 
     fn handle_preparing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                    view: &ViewInfo,
-                                   msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {
+                                   header: &Header, msg: &ConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {
         let node_id = node.id();
 
         let seq = deciding_log.sequence_number();
@@ -152,11 +152,12 @@ impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
 
     fn handle_committing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                        view: &ViewInfo,
-                                       msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
+                                       header: &Header, msg: &ConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 
     fn handle_committing_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                     view: &ViewInfo,
-                                    msg: StoredConsensusMessage<D::Request>, node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
+                                    header: &Header, msg: &ConsensusMessage<D::Request>,
+                                    node: &NT) where NT: OrderProtocolSendNode<D, PBFT<D>> {}
 }
 
 impl<D> ReplicaAccessory<D>
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 8c9bb223..9b7cefe1 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -20,12 +20,11 @@ use atlas_metrics::metrics::metric_duration;
 
 use crate::bft::consensus::accessory::{AccessoryConsensus, ConsensusDecisionAccessory};
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
-use crate::bft::log::deciding::WorkingDecisionLog;
+use crate::bft::log::deciding::{CompletedBatch, WorkingDecisionLog};
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{ConsensusMetrics, PRE_PREPARE_ANALYSIS_ID};
 use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::deciding_log::{CompletedBatch, DecidingLog};
 use crate::bft::msg_log::decisions::{IncompleteProof, StoredConsensusMessage};
 use crate::bft::PBFT;
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer};
@@ -95,7 +94,7 @@ pub enum DecisionStatus<O> {
     /// THe second Vec<Digest> is a vec with digests of the requests contained in the batch
     /// The third is the messages that should be persisted for this batch to be considered persisted
     Decided(StoredMessage<ConsensusMessage<O>>),
-    DecidedIgnored
+    DecidedIgnored,
 }
 
 /// A message queue for this particular consensus instance
@@ -175,7 +174,7 @@ impl<O> MessageQueue<O> {
 }
 
 impl<D, PL> ConsensusDecision<D, PL>
-    where D: ApplicationData + 'static,{
+    where D: ApplicationData + 'static, {
     pub fn init_decision(node_id: NodeId, seq_no: SeqNo, view: &ViewInfo, persistent_log: PL) -> Self {
         Self {
             node_id,
@@ -344,7 +343,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                 let batch_metadata = self.working_log.process_pre_prepare(header.clone(), &message,
                                                                           header.digest().clone(), digests)?;
 
-                let mut result ;
+                let mut result;
 
                 self.phase = if received == view.leader_set().len() {
                     let batch_metadata = batch_metadata.unwrap();
@@ -371,7 +370,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     log.all_batches_received(batch_metadata);
 
                     self.accessory.handle_pre_prepare_phase_completed(&self.working_log,
-                                                                      &view, stored_msg.clone(), node);
+                                                                      &view, &header, &message, node);
 
                     self.message_queue.signal();
 
@@ -385,8 +384,8 @@ impl<D, PL> ConsensusDecision<D, PL>
                     debug!("{:?} // Received pre prepare message {:?} from {:?}. Current received {:?}",
                         self.node_id, stored_msg.message(), stored_msg.header().from(), received);
 
-                    self.accessory.handle_partial_pre_prepare(&self.working_log,
-                                                              &view, stored_msg.clone(), &**node);
+                    self.accessory.handle_partial_pre_prepare(&self.working_log, &view,
+                                                              &header, &message, &**node);
 
                     result = DecisionStatus::Deciding(StoredMessage::new(header, message));
 
@@ -457,7 +456,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     let current_digest = self.working_log.current_digest().unwrap();
 
                     self.accessory.handle_preparing_quorum(&self.working_log, &view,
-                                                           stored_msg.clone(), &**node);
+                                                           &header, &message, &**node);
 
                     self.message_queue.signal();
 
@@ -469,7 +468,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         self.node_id, stored_msg.message().sequence_number(), header.from(), received);
 
                     self.accessory.handle_preparing_no_quorum(&self.working_log, &view,
-                                                              stored_msg.clone(), &**node);
+                                                              &header, &message, &**node);
 
                     result = DecisionStatus::Deciding(StoredMessage::new(header, message));
 
@@ -528,7 +527,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.consensus_metrics.commit_quorum_recvd();
 
                     self.accessory.handle_committing_quorum(&self.working_log, &view,
-                                                            stored_msg.clone(), &**node);
+                                                            &header, &message, &**node);
 
                     Ok(DecisionStatus::Decided(StoredMessage::new(header, message)))
                 } else {
@@ -538,7 +537,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.phase = DecisionPhase::Committing(received);
 
                     self.accessory.handle_committing_no_quorum(&self.working_log, &view,
-                                                               stored_msg.clone(), &**node);
+                                                               &header, &message, &**node);
 
                     Ok(DecisionStatus::Deciding(StoredMessage::new(header, message)))
                 };
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 135a7d83..29e3e2ba 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -23,11 +23,11 @@ use atlas_metrics::metrics::metric_increment;
 
 use crate::bft::{PBFT, SysMsg};
 use crate::bft::consensus::decision::{ConsensusDecision, DecisionPollStatus, DecisionStatus, MessageQueue};
+use crate::bft::log::deciding::CompletedBatch;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::OPERATIONS_PROCESSED_ID;
 use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::deciding_log::CompletedBatch;
 use crate::bft::msg_log::decisions::{DecisionLog, IncompleteProof, Proof};
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer};
 use crate::bft::sync::view::ViewInfo;
@@ -440,6 +440,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             DecisionStatus::Decided(message) => {
                 ConsensusStatus::Decided
             }
+            DecisionStatus::DecidedIgnored => {}
             DecisionStatus::MessageIgnored => {
                 ConsensusStatus::MessageIgnored
             }
diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
index 9104f6a7..72eecf6a 100644
--- a/febft-pbft-consensus/src/bft/log/deciding/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -16,6 +16,8 @@ use crate::bft::msg_log::deciding_log::{DecidingLog, FullBatch};
 use crate::bft::msg_log::decisions::{IncompleteProof, ProofMetadata};
 use crate::bft::sync::view::ViewInfo;
 
+/// Information about the completed batch, the contained requests and
+/// other relevant information
 pub struct CompletedBatch<O> {
     seq: SeqNo,
     // The overall digest of the entire batch
@@ -26,6 +28,7 @@ pub struct CompletedBatch<O> {
     client_request_info: Vec<ClientRqInfo>,
     // The client requests contained in this batch
     client_requests: Vec<StoredRequestMessage<O>>,
+    // The metadata for the batch
     batch_meta: BatchMeta,
 }
 
@@ -243,7 +246,7 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
     }
 }
 
-impl Orderable for WorkingDecisionLog {
+impl<O> Orderable for WorkingDecisionLog<O> {
     fn sequence_number(&self) -> SeqNo {
         self.seq_no
     }

From 9fa9522486f2f4bd5cf48f7eb260b8e60031c56a Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 4 Oct 2023 11:00:26 +0100
Subject: [PATCH 54/80] Remapped the return objects for the ordering protocol.
 Introduced MaybeVec for situations where it could be one item or a vec of
 items. Started working on adapting the replicas to the new functioning.

---
 febft-pbft-consensus/src/bft/mod.rs | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index c1f19f80..c9aa28de 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -8,6 +8,7 @@ use std::ops::Drop;
 use std::sync::Arc;
 use std::sync::atomic::AtomicBool;
 use std::time::Instant;
+use ::log::{debug, info, trace, warn};
 
 use log::{debug, info, trace, warn};
 
@@ -131,14 +132,14 @@ impl<D, NT, PL> OrderProtocolTolerance for PBFTOrderProtocol<D, NT, PL>
     }
 }
 
-impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
+impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
           PL: Clone {
     type Serialization = PBFTConsensus<D>;
     type Config = PBFTConfig<D>;
 
-    fn initialize(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT, PL>) -> Result<Self> where
+    fn initialize(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT>) -> Result<Self> where
         Self: Sized,
     {
         Self::initialize_protocol(config, args, None)
@@ -195,7 +196,7 @@ impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
         }
     }
 
-    fn process_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
+    fn process_message(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OrderProtocolExecResult<D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         match self.phase {
             ConsensusPhase::NormalPhase => {
@@ -288,7 +289,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
           PL: Clone {
-    fn initialize_protocol(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT, PL>,
+    fn initialize_protocol(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT>,
                            initial_state: Option<DecisionLog<D::Request>>) -> Result<Self> {
         let PBFTConfig {
             node_id,
@@ -359,7 +360,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         match poll_result {
             SynchronizerPollStatus::Recv => OrderProtocolPoll::ReceiveFromReplicas,
             SynchronizerPollStatus::NextMessage(h, m) => {
-                OrderProtocolPoll::Exec(StoredMessage::new(h, Protocol::new(PBFTMessage::ViewChange(m))))
+                OrderProtocolPoll::Exec(StoredMessage::new(h, PBFTMessage::ViewChange(m)))
             }
             SynchronizerPollStatus::ResumeViewChange => {
                 debug!("{:?} // Resuming view change", self.node.id());
@@ -444,7 +445,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         match polled_message {
             ConsensusPollStatus::Recv => OrderProtocolPoll::ReceiveFromReplicas,
             ConsensusPollStatus::NextMessage(h, m) => {
-                OrderProtocolPoll::Exec(StoredMessage::new(h, Protocol::new(PBFTMessage::Consensus(m))))
+                OrderProtocolPoll::Exec(StoredMessage::new(h, PBFTMessage::Consensus(m)))
             }
             ConsensusPollStatus::Decided => {
                 return OrderProtocolPoll::Decided(self.finalize_all_possible().expect("Failed to finalize decisions"));
@@ -452,7 +453,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         }
     }
 
-    fn update_sync_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
+    fn update_sync_phase(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OrderProtocolExecResult<D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let (header, protocol) = message.into_inner();
 
@@ -496,11 +497,11 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         Ok(OrderProtocolExecResult::Success)
     }
 
-    fn update_normal_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
+    fn update_normal_phase(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OrderProtocolExecResult<D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let (header, protocol) = message.into_inner();
 
-        match protocol.into_inner() {
+        match protocol {
             PBFTMessage::Consensus(message) => {
                 return self.adv_consensus(header, message);
             }

From 00042e113ed968dfe0fd266fb9d1ab88fc3445c7 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 5 Oct 2023 19:51:56 +0100
Subject: [PATCH 55/80] Decision log should return the decisions that have been
 successfully added to the log in order for the replica to coordinate
 checkpoints and general keeping track. The decision log can still decide to
 keep the responsibility of delivery the batches himself to the executor, to
 support certain asynchronous tasks.

---
 febft-pbft-consensus/src/bft/mod.rs           | 19 ++++++++++---------
 .../src/bft/msg_log/decided_log/mod.rs        |  4 ++--
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index c1f19f80..c9aa28de 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -8,6 +8,7 @@ use std::ops::Drop;
 use std::sync::Arc;
 use std::sync::atomic::AtomicBool;
 use std::time::Instant;
+use ::log::{debug, info, trace, warn};
 
 use log::{debug, info, trace, warn};
 
@@ -131,14 +132,14 @@ impl<D, NT, PL> OrderProtocolTolerance for PBFTOrderProtocol<D, NT, PL>
     }
 }
 
-impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
+impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
           PL: Clone {
     type Serialization = PBFTConsensus<D>;
     type Config = PBFTConfig<D>;
 
-    fn initialize(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT, PL>) -> Result<Self> where
+    fn initialize(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT>) -> Result<Self> where
         Self: Sized,
     {
         Self::initialize_protocol(config, args, None)
@@ -195,7 +196,7 @@ impl<D, NT, PL> OrderingProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
         }
     }
 
-    fn process_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
+    fn process_message(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OrderProtocolExecResult<D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         match self.phase {
             ConsensusPhase::NormalPhase => {
@@ -288,7 +289,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
           PL: Clone {
-    fn initialize_protocol(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT, PL>,
+    fn initialize_protocol(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT>,
                            initial_state: Option<DecisionLog<D::Request>>) -> Result<Self> {
         let PBFTConfig {
             node_id,
@@ -359,7 +360,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         match poll_result {
             SynchronizerPollStatus::Recv => OrderProtocolPoll::ReceiveFromReplicas,
             SynchronizerPollStatus::NextMessage(h, m) => {
-                OrderProtocolPoll::Exec(StoredMessage::new(h, Protocol::new(PBFTMessage::ViewChange(m))))
+                OrderProtocolPoll::Exec(StoredMessage::new(h, PBFTMessage::ViewChange(m)))
             }
             SynchronizerPollStatus::ResumeViewChange => {
                 debug!("{:?} // Resuming view change", self.node.id());
@@ -444,7 +445,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         match polled_message {
             ConsensusPollStatus::Recv => OrderProtocolPoll::ReceiveFromReplicas,
             ConsensusPollStatus::NextMessage(h, m) => {
-                OrderProtocolPoll::Exec(StoredMessage::new(h, Protocol::new(PBFTMessage::Consensus(m))))
+                OrderProtocolPoll::Exec(StoredMessage::new(h, PBFTMessage::Consensus(m)))
             }
             ConsensusPollStatus::Decided => {
                 return OrderProtocolPoll::Decided(self.finalize_all_possible().expect("Failed to finalize decisions"));
@@ -452,7 +453,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         }
     }
 
-    fn update_sync_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
+    fn update_sync_phase(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OrderProtocolExecResult<D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let (header, protocol) = message.into_inner();
 
@@ -496,11 +497,11 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         Ok(OrderProtocolExecResult::Success)
     }
 
-    fn update_normal_phase(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>) -> Result<OrderProtocolExecResult<D::Request>>
+    fn update_normal_phase(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OrderProtocolExecResult<D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let (header, protocol) = message.into_inner();
 
-        match protocol.into_inner() {
+        match protocol {
             PBFTMessage::Consensus(message) => {
                 return self.adv_consensus(header, message);
             }
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
index 50bb3224..5eed7416 100755
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
@@ -159,8 +159,8 @@ impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
     /// Basically persists the metadata for a given consensus num
     pub fn all_batches_received(&mut self, metadata: ProofMetadata)
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        self.persistent_log.write_proof_metadata(OperationMode::NonBlockingSync(None),
-                                                 metadata).unwrap();
+        self.persistent_log.write_decision_metadata(OperationMode::NonBlockingSync(None),
+                                                    metadata).unwrap();
     }
 
     /// Finalize a batch of client requests decided on the consensus instance

From 3b365db9fa268a21f45aa44f766386ea90e32e11 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Sat, 7 Oct 2023 00:37:16 +0100
Subject: [PATCH 56/80] Work more on the decision log. Fix some things with
 state messages are handled to support the new phased approach. Still have to
 adapt the ordering protocol and the log transfer protocol to support the new
 features.

---
 .../src/bft/consensus/accessory/mod.rs        |  20 +-
 .../bft/consensus/accessory/replica/mod.rs    |   1 -
 .../src/bft/consensus/decision/mod.rs         |   4 +-
 febft-pbft-consensus/src/bft/consensus/mod.rs |   5 +-
 .../src/bft/log/decided/mod.rs                |   6 +-
 .../src/bft/log/deciding/mod.rs               |   2 +-
 .../src/bft/{msg_log => log}/decisions/mod.rs |  10 +-
 .../src/bft/{msg_log => log}/mod.rs           |  17 +-
 febft-pbft-consensus/src/bft/mod.rs           |   5 +-
 .../src/bft/msg_log/decided_log/mod.rs        | 275 --------
 .../src/bft/msg_log/deciding_log/mod.rs       | 588 ------------------
 11 files changed, 28 insertions(+), 905 deletions(-)
 rename febft-pbft-consensus/src/bft/{msg_log => log}/decisions/mod.rs (98%)
 rename febft-pbft-consensus/src/bft/{msg_log => log}/mod.rs (50%)
 delete mode 100755 febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
 delete mode 100644 febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs

diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
index cc1aa6bc..bf4a0bf0 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
@@ -9,8 +9,6 @@ use crate::bft::consensus::accessory::replica::ReplicaAccessory;
 use crate::bft::log::deciding::WorkingDecisionLog;
 use crate::bft::message::ConsensusMessage;
 use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::msg_log::deciding_log::DecidingLog;
-use crate::bft::msg_log::decisions::StoredConsensusMessage;
 use crate::bft::sync::view::ViewInfo;
 
 pub mod replica;
@@ -21,7 +19,7 @@ pub enum ConsensusDecisionAccessory<D>
     Replica(ReplicaAccessory<D>),
 }
 
-pub trait AccessoryConsensus<D> where D: ApplicationData + 'static,{
+pub trait AccessoryConsensus<D> where D: ApplicationData + 'static, {
     /// Handle the reception of a pre-prepare message without having completed the pre prepare phase
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
@@ -63,7 +61,7 @@ pub trait AccessoryConsensus<D> where D: ApplicationData + 'static,{
 }
 
 impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
-    where D: ApplicationData + 'static{
+    where D: ApplicationData + 'static {
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
                                       header: &Header, msg: &ConsensusMessage<D::Request>,
@@ -71,7 +69,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_partial_pre_prepare(deciding_log, view, msg, node);
+                rep.handle_partial_pre_prepare(deciding_log, view, header, msg, node);
             }
         }
     }
@@ -79,11 +77,11 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                               view: &ViewInfo,
                                               header: &Header, msg: &ConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static{
+                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_pre_prepare_phase_completed(deciding_log, view, msg, node);
+                rep.handle_pre_prepare_phase_completed(deciding_log, view, header, msg, node);
             }
         }
     }
@@ -95,7 +93,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_preparing_no_quorum(deciding_log, view, msg, node);
+                rep.handle_preparing_no_quorum(deciding_log, view, header, msg, node);
             }
         }
     }
@@ -107,7 +105,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_preparing_quorum(deciding_log, view, msg, node);
+                rep.handle_preparing_quorum(deciding_log, view, header, msg, node);
             }
         }
     }
@@ -119,7 +117,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_committing_no_quorum(deciding_log, view, msg, node);
+                rep.handle_committing_no_quorum(deciding_log, view, header, msg, node);
             }
         }
     }
@@ -131,7 +129,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_committing_quorum(deciding_log, view, msg, node);
+                rep.handle_committing_quorum(deciding_log, view, header, msg, node);
             }
         }
     }
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index a25f58d7..cba7cfb6 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -16,7 +16,6 @@ use crate::bft::{PBFT, SysMsg};
 use crate::bft::consensus::accessory::AccessoryConsensus;
 use crate::bft::log::deciding::WorkingDecisionLog;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
-use crate::bft::msg_log::decisions::StoredConsensusMessage;
 use crate::bft::sync::view::ViewInfo;
 
 pub struct ReplicaAccessory<D>
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 9b7cefe1..2dedbfb2 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -7,7 +7,6 @@ use chrono::Utc;
 use log::{debug, info, warn};
 
 use atlas_common::error::*;
-use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
@@ -21,11 +20,10 @@ use atlas_metrics::metrics::metric_duration;
 use crate::bft::consensus::accessory::{AccessoryConsensus, ConsensusDecisionAccessory};
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
 use crate::bft::log::deciding::{CompletedBatch, WorkingDecisionLog};
+use crate::bft::log::decisions::IncompleteProof;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{ConsensusMetrics, PRE_PREPARE_ANALYSIS_ID};
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::{IncompleteProof, StoredConsensusMessage};
 use crate::bft::PBFT;
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer};
 use crate::bft::sync::view::ViewInfo;
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 29e3e2ba..b55e3878 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -15,7 +15,7 @@ use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
+use atlas_core::persistent_log::{OrderingProtocolLog};
 use atlas_core::timeouts::Timeouts;
 use atlas_smr_application::ExecutorHandle;
 use atlas_smr_application::serialize::ApplicationData;
@@ -24,11 +24,10 @@ use atlas_metrics::metrics::metric_increment;
 use crate::bft::{PBFT, SysMsg};
 use crate::bft::consensus::decision::{ConsensusDecision, DecisionPollStatus, DecisionStatus, MessageQueue};
 use crate::bft::log::deciding::CompletedBatch;
+use crate::bft::log::decisions::IncompleteProof;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::OPERATIONS_PROCESSED_ID;
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::{DecisionLog, IncompleteProof, Proof};
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer};
 use crate::bft::sync::view::ViewInfo;
 
diff --git a/febft-pbft-consensus/src/bft/log/decided/mod.rs b/febft-pbft-consensus/src/bft/log/decided/mod.rs
index 5e9aeb75..019461a5 100644
--- a/febft-pbft-consensus/src/bft/log/decided/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decided/mod.rs
@@ -1,4 +1,4 @@
-use crate::bft::msg_log::decisions::Proof;
+use crate::bft::log::decisions::Proof;
 
 /// A necessary decision log for the ability to perform view changes.
 /// Only stores the latest performed decision
@@ -15,10 +15,12 @@ impl<O> OngoingDecisionLog<O> {
         }
     }
 
+    /// Install a given proof
     fn install_proof(&mut self, proof: Proof<O>) {
         self.last_decision = Some(proof)
     }
-    
+
+    /// Get the last decision
     fn last_decision(&self) -> Option<Proof<O>>  {
         self.last_decision.clone()
     }
diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
index 72eecf6a..23e4f2e0 100644
--- a/febft-pbft-consensus/src/bft/log/deciding/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -160,7 +160,7 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
 
             self.batch_digest = Some(digest.clone());
 
-            Some(ProofMetadata::new(self.seq_no, digest, ordering))
+            Some(ProofMetadata::new(self.seq_no, digest, ordering, self.current_batch_size))
         } else {
             None
         })
diff --git a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
similarity index 98%
rename from febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
rename to febft-pbft-consensus/src/bft/log/decisions/mod.rs
index 4681306c..35e19433 100644
--- a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
@@ -12,7 +12,7 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::StoredMessage;
 use atlas_core::ordering_protocol::networking::serialize::{OrderProtocolLog, OrderProtocolProof};
 
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
 use crate::bft::msg_log::deciding_log::CompletedBatch;
 
 pub type StoredConsensusMessage<O> = Arc<ReadOnly<StoredMessage<ConsensusMessage<O>>>>;
@@ -47,6 +47,7 @@ pub struct ProofMetadata {
     seq_no: SeqNo,
     batch_digest: Digest,
     pre_prepare_ordering: Vec<Digest>,
+    contained_client_rqs: usize
 }
 
 impl Orderable for ProofMetadata {
@@ -100,11 +101,12 @@ impl PrepareSet {
 
 impl ProofMetadata {
     /// Create a new proof metadata
-    pub(crate) fn new(seq_no: SeqNo, digest: Digest, pre_prepare_ordering: Vec<Digest>) -> Self {
+    pub(crate) fn new(seq_no: SeqNo, digest: Digest, pre_prepare_ordering: Vec<Digest>, contained_rqs: usize) -> Self {
         Self {
             seq_no,
             batch_digest: digest,
             pre_prepare_ordering,
+            contained_client_rqs: contained_rqs
         }
     }
 
@@ -119,6 +121,10 @@ impl ProofMetadata {
     pub fn pre_prepare_ordering(&self) -> &Vec<Digest> {
         &self.pre_prepare_ordering
     }
+
+    pub fn contained_client_rqs(&self) -> usize {
+        self.contained_client_rqs
+    }
 }
 
 impl<O> Proof<O> {
diff --git a/febft-pbft-consensus/src/bft/msg_log/mod.rs b/febft-pbft-consensus/src/bft/log/mod.rs
similarity index 50%
rename from febft-pbft-consensus/src/bft/msg_log/mod.rs
rename to febft-pbft-consensus/src/bft/log/mod.rs
index c6dd978a..b417e083 100644
--- a/febft-pbft-consensus/src/bft/msg_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/mod.rs
@@ -1,24 +1,11 @@
-//! A module to manage the `febft` message log.
-
-use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::SeqNo;
 use atlas_communication::message::Header;
 use atlas_core::messages::RequestMessage;
-use atlas_smr_application::serialize::ApplicationData;
-
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::DecisionLog;
 
+pub mod decided;
+pub mod deciding;
 pub mod decisions;
-pub mod deciding_log;
-pub mod decided_log;
-
-pub fn initialize_decided_log<D: ApplicationData + 'static, PL>(node_id: NodeId,
-                                                                persistent_log: PL,
-                                                                state: Option<DecisionLog<D::Request>>) -> Result<Log<D, PL>> {
-    Ok(Log::init_decided_log(node_id, persistent_log, state))
-}
 
 #[inline]
 pub fn operation_key<O>(header: &Header, message: &RequestMessage<O>) -> u64 {
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index c9aa28de..7bfb0e11 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -48,10 +48,7 @@ pub mod consensus;
 pub mod proposer;
 pub mod sync;
 pub mod msg_log;
-pub mod log {
-    pub mod deciding;
-    pub mod decided;
-}
+pub mod log;
 pub mod config;
 pub mod message;
 pub mod observer;
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
deleted file mode 100755
index 5eed7416..00000000
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ /dev/null
@@ -1,275 +0,0 @@
-use log::error;
-
-use atlas_common::error::*;
-use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_core::ordering_protocol::{DecisionInformation, ProtocolConsensusDecision};
-use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_smr_application::app::UpdateBatch;
-use atlas_smr_application::serialize::ApplicationData;
-
-use crate::bft::message::ConsensusMessageKind;
-use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::msg_log::deciding_log::CompletedBatch;
-use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata, StoredConsensusMessage};
-use crate::bft::msg_log::operation_key;
-use crate::bft::sync::view::ViewInfo;
-
-/// The log of decisions that have already been processed by the consensus
-/// algorithm
-pub struct Log<D, PL> where D: ApplicationData + 'static {
-    // The log for all of the already decided consensus instances
-    dec_log: DecisionLog<D::Request>,
-
-    // A handle to the persistent log
-    persistent_log: PL,
-}
-
-impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
-    pub(crate) fn init_decided_log(node_id: NodeId, persistent_log: PL,
-                                   dec_log: Option<DecisionLog<D::Request>>) -> Self {
-        Self {
-            dec_log: dec_log.unwrap_or(DecisionLog::new()),
-            persistent_log,
-        }
-    }
-
-    /// Returns a reference to a subset of this log, containing only
-    /// consensus messages.
-    pub fn decision_log(&self) -> &DecisionLog<D::Request> {
-        &self.dec_log
-    }
-
-    pub fn mut_decision_log(&mut self) -> &mut DecisionLog<D::Request> {
-        &mut self.dec_log
-    }
-
-    /// Read the current state, if existent, from the persistent storage
-    ///
-    /// FIXME: The view initialization might have to be changed if we want to introduce reconfiguration
-    pub fn read_current_state(&self, n: usize, f: usize) -> Result<Option<(ViewInfo, DecisionLog<D::Request>)>>
-        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>, PBFTConsensus<D>> {
-        let option = self.persistent_log.read_state(OperationMode::BlockingSync)?;
-
-        if let Some((view, dec_log)) = option {
-            Ok(Some((view, dec_log)))
-        } else {
-            Ok(None)
-        }
-    }
-
-    /// Take a snapshot of the log, used to recover a replica.
-    ///
-    /// This method may fail if we are waiting for the latest application
-    /// state to be returned by the execution layer.
-    ///
-    pub fn snapshot(&self, view: ViewInfo) -> Result<(ViewInfo, DecisionLog<D::Request>)> {
-        Ok((view, self.dec_log.clone()))
-    }
-
-    /// Insert a consensus message into the log.
-    /// We can use this method when we want to prevent a clone, as this takes
-    /// just a reference.
-    /// This is mostly used for pre prepares as they contain all the requests and are therefore very expensive to send
-    pub fn insert_consensus(
-        &mut self,
-        consensus_msg: StoredConsensusMessage<D::Request>,
-    ) where PL: OrderingProtocolLog<D, PBFTConsensus<D>>,
-    {
-        if let Err(err) = self
-            .persistent_log
-            .write_message(OperationMode::NonBlockingSync(None), consensus_msg)
-        {
-            error!("Failed to persist message {:?}", err);
-        }
-    }
-
-    /// Install a proof of a consensus instance into the log.
-    /// This is done when we receive the final SYNC message from the leader
-    /// which contains all of the collects
-    /// If we are missing the request determined by the
-    pub fn install_proof(&mut self, seq: SeqNo, proof: Proof<D::Request>) -> Result<ProtocolConsensusDecision<D::Request>>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        let batch_execution_info = ProtocolConsensusDecision::from(&proof);
-
-        if let Some(decision) = self.decision_log().last_decision() {
-            if decision.seq_no() == seq {
-                // Well well well, if it isn't what I'm trying to add?
-                //This should not be possible
-
-                return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidedLog,
-                                                  "Already have decision at that seq no"));
-            } else {
-                self.mut_decision_log().append_proof(proof.clone());
-            }
-        }
-
-        if let Err(err) = self.persistent_log
-            .write_proof(OperationMode::NonBlockingSync(None), proof) {
-            error!("Failed to persist proof {:?}", err);
-        }
-
-        Ok(batch_execution_info)
-    }
-
-    /// Clear the occurrences of a seq no from the decision log
-    pub fn clear_last_occurrence(&mut self, seq: SeqNo)
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        if let Err(err) = self.persistent_log.write_invalidate(OperationMode::NonBlockingSync(None), seq) {
-            error!("Failed to invalidate last occurrence {:?}", err);
-        }
-    }
-
-    /// Update the log state, received from the CST protocol.
-    pub fn install_state(&mut self, view: ViewInfo, dec_log: DecisionLog<D::Request>)
-        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>, PBFTConsensus<D>> {
-
-        //Replace the log
-        self.dec_log = dec_log.clone();
-
-        let last_seq = self.dec_log.last_execution().unwrap_or(SeqNo::ZERO);
-
-        if let Err(err) = self.persistent_log
-            .write_install_state(OperationMode::NonBlockingSync(None), view, dec_log) {
-            error!("Failed to persist message {:?}", err);
-        }
-    }
-
-    /// End the state of an on-going checkpoint.
-    ///
-    /// This method should only be called when `finalize_request()` reports
-    /// `Info::BeginCheckpoint`, and the requested application state is received
-    /// on the core server task's master channel.
-    pub fn finalize_checkpoint(&mut self, final_seq: SeqNo) -> Result<()> {
-        let mut decided_request_count;
-
-        //Clear the log of messages up to final_seq.
-        //Messages ahead of final_seq will not be removed as they are not included in the
-        //Checkpoint and therefore must be logged.
-        {
-            let mut guard = &mut self.dec_log;
-
-            decided_request_count = guard.clear_until_seq(final_seq);
-        }
-
-        Ok(())
-    }
-
-    /// Register that all the batches for a given decision have already been received
-    /// Basically persists the metadata for a given consensus num
-    pub fn all_batches_received(&mut self, metadata: ProofMetadata)
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        self.persistent_log.write_decision_metadata(OperationMode::NonBlockingSync(None),
-                                                    metadata).unwrap();
-    }
-
-    /// Finalize a batch of client requests decided on the consensus instance
-    /// with sequence number `seq`, retrieving the payload associated with their
-    /// given digests `digests`.
-    ///
-    /// The decided log may be cleared resulting from this operation. Check the enum variant of
-    /// `Info`, to perform a local checkpoint when appropriate.
-    ///
-    /// Returns a [`Option::None`] when we are running in Strict mode, indicating the
-    /// batch request has been put in the execution queue, waiting for all of the messages
-    /// to be persisted
-    pub fn finalize_batch(
-        &mut self,
-        seq: SeqNo,
-        completed_batch: CompletedBatch<D::Request>,
-    ) -> Result<ProtocolConsensusDecision<D::Request>> {
-        //println!("Finalized batch of OPS seq {:?} on Node {:?}", seq, self.node_id);
-
-        let batch = {
-            let mut batch = UpdateBatch::new_with_cap(seq, completed_batch.request_count());
-
-            for message in completed_batch.pre_prepare_messages() {
-                let reqs = {
-                    if let ConsensusMessageKind::PrePrepare(reqs) = (*message.message().kind()).clone() {
-                        reqs
-                    } else { unreachable!() }
-                };
-
-                for (header, message) in reqs.into_iter()
-                    .map(|x| x.into_inner()) {
-                    let _key = operation_key::<D::Request>(&header, &message);
-
-                    //TODO: Maybe make this run on separate thread?
-                    // let seq_no = latest_op_guard
-                    //     .get(key)
-                    //     .unwrap_or(&SeqNo::ZERO);
-                    //
-                    // if message.sequence_number() > *seq_no {
-                    //     latest_op_guard.insert(key, message.sequence_number());
-                    // }
-
-                    batch.add(
-                        header.from(),
-                        message.session_id(),
-                        message.sequence_number(),
-                        message.into_inner_operation(),
-                    );
-                }
-            }
-
-            batch.append_batch_meta(completed_batch.batch_meta().clone());
-
-            batch
-        };
-
-        // the last executed sequence number
-        let f = 1;
-
-        // Finalize the execution and store the proof in the log as a proof
-        // instead of an ongoing decision
-        self.dec_log.finished_quorum_execution(&completed_batch, seq, f)?;
-
-        let decision = ProtocolConsensusDecision::new(seq, batch,
-                                                      Some(DecisionInformation::from(completed_batch)));
-
-        Ok(decision)
-    }
-
-    /// Collects the most up to date data we have in store.
-    /// Accepts the f for the view that it is looking for
-    /// It must accept this f as the reconfiguration of the network
-    /// can alter the f from one seq no to the next
-    pub fn last_proof(&self, f: usize) -> Option<Proof<D::Request>> {
-        self.dec_log.last_decision()
-    }
-}
-
-impl<O> From<&Proof<O>> for ProtocolConsensusDecision<O> where O: Clone {
-    fn from(value: &Proof<O>) -> Self {
-        let mut update_batch = UpdateBatch::new(value.seq_no());
-
-        if !value.are_pre_prepares_ordered().unwrap() {
-            //The batch should be provided to this already ordered.
-            todo!()
-        }
-
-        for pre_prepare in value.pre_prepares() {
-            let consensus_msg = (*pre_prepare.message()).clone();
-
-            let reqs = match consensus_msg.into_kind() {
-                ConsensusMessageKind::PrePrepare(reqs) => { reqs }
-                _ => {
-                    unreachable!()
-                }
-            };
-
-            for request in reqs {
-                let (header, message) = request.into_inner();
-
-                update_batch.add(header.from(),
-                                 message.session_id(),
-                                 message.sequence_number(),
-                                 message.into_inner_operation());
-            }
-        }
-
-        ProtocolConsensusDecision::new(value.seq_no(),
-                                       update_batch,
-                                       None)
-    }
-}
\ No newline at end of file
diff --git a/febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs
deleted file mode 100644
index cb01f717..00000000
--- a/febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs
+++ /dev/null
@@ -1,588 +0,0 @@
-use std::cmp::Ordering;
-use std::collections::{BTreeMap, BTreeSet};
-use std::iter;
-use std::iter::zip;
-use std::sync::{Arc, Mutex};
-use std::time::Instant;
-
-use atlas_common::crypto::hash::{Context, Digest};
-use atlas_common::error::*;
-use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_core::messages::ClientRqInfo;
-use atlas_core::ordering_protocol::DecisionInformation;
-use atlas_metrics::benchmarks::BatchMeta;
-use atlas_metrics::metrics::metric_duration;
-
-use crate::bft::message::ConsensusMessageKind;
-use crate::bft::metric::PRE_PREPARE_LOG_ANALYSIS_ID;
-use crate::bft::msg_log::decisions::{IncompleteProof, PrepareSet, Proof, ProofMetadata, StoredConsensusMessage, ViewDecisionPair};
-use crate::bft::sync::view::ViewInfo;
-
-/// A batch that has been decided by the consensus instance and is now ready to be delivered to the
-/// Executor for execution.
-/// Contains all of the necessary information for when we are using the strict persistency mode
-pub struct CompletedBatch<O> {
-    // The sequence number of the batch
-    seq_no: SeqNo,
-
-    //The digest of the batch
-    batch_digest: Digest,
-    // The ordering of the pre prepares
-    pre_prepare_ordering: Vec<Digest>,
-    // The prepare message of the batch
-    pre_prepare_messages: Vec<StoredConsensusMessage<O>>,
-    // The prepare messages for this batch
-    prepare_messages: Vec<StoredConsensusMessage<O>>,
-    // The commit messages for this batch
-    commit_messages: Vec<StoredConsensusMessage<O>>,
-    // The information of the client requests that are contained in this batch
-    client_requests: Vec<ClientRqInfo>,
-    //The messages that must be persisted for this consensus decision to be executable
-    //This should contain the pre prepare, quorum of prepares and quorum of commits
-    messages_to_persist: Vec<Digest>,
-
-    // The metadata for this batch (mostly statistics)
-    batch_meta: BatchMeta,
-}
-
-pub struct DecidingLog<O> {
-    node_id: NodeId,
-    seq_no: SeqNo,
-
-    // Detect duplicate requests sent by replicas
-    duplicate_detection: DuplicateReplicaEvaluator,
-    //The digest of the entire batch that is currently being processed
-    // This will only be calculated when we receive all of the requests necessary
-    // As this digest requires the knowledge of all of them
-    current_digest: Option<Digest>,
-    // How many pre prepares have we received
-    current_received_pre_prepares: usize,
-    //The size of batch that is currently being processed. Increases as we receive more pre prepares
-    current_batch_size: usize,
-    //The client requests that are currently being processed
-    //Does not have to follow the correct order, only has to contain the requests
-    client_rqs: Vec<ClientRqInfo>,
-
-    // The message log of the current ongoing decision
-    ongoing_decision: OnGoingDecision<O>,
-
-    // The set of leaders that is currently in vigour for this consensus decision
-    leader_set: Vec<NodeId>,
-    // Which hash space should each leader be responsible for
-    request_space_slices: BTreeMap<NodeId, (Vec<u8>, Vec<u8>)>,
-
-    //A list of digests of all consensus related messages pertaining to this
-    //Consensus instance. Used to keep track of if the persistent log has saved the messages already
-    //So the requests can be executed
-    current_messages_to_persist: Vec<Digest>,
-
-    // Some logging information about metadata
-    batch_meta: Arc<Mutex<BatchMeta>>,
-}
-
-/// Checks to make sure replicas aren't providing more than one vote for the
-/// Same consensus decision
-#[derive(Default)]
-pub struct DuplicateReplicaEvaluator {
-    // The set of leaders that is currently in vigour for this consensus decision
-    leader_set: Vec<NodeId>,
-    // The set of leaders that have already sent a pre prepare message
-    received_pre_prepare_messages: BTreeSet<NodeId>,
-    // The set of leaders that have already sent a prepare message
-    received_prepare_messages: BTreeSet<NodeId>,
-    // The set of leaders that have already sent a commit message
-    received_commit_messages: BTreeSet<NodeId>,
-}
-
-/// Store the messages corresponding to a given ongoing consensus decision
-pub struct OnGoingDecision<O> {
-    pre_prepare_digests: Vec<Option<Digest>>,
-    // This must be a vec of options since we want to keep the ordering of the pre prepare messages
-    // Even when the messages have not yet been received
-    pre_prepare_messages: Vec<Option<StoredConsensusMessage<O>>>,
-    prepare_messages: Vec<StoredConsensusMessage<O>>,
-    commit_messages: Vec<StoredConsensusMessage<O>>,
-}
-
-/// Information about a full batch
-pub type FullBatch = ProofMetadata;
-
-impl<O> DecidingLog<O> {
-    pub fn new(node_id: NodeId, seq_no: SeqNo, view: &ViewInfo) -> Self {
-        Self {
-            node_id,
-            seq_no,
-            duplicate_detection: Default::default(),
-            current_digest: None,
-            current_received_pre_prepares: 0,
-            ongoing_decision: OnGoingDecision::initialize(view.leader_set().len()),
-            leader_set: view.leader_set().clone(),
-            request_space_slices: view.hash_space_division().clone(),
-            current_batch_size: 0,
-            current_messages_to_persist: vec![],
-            batch_meta: Arc::new(Mutex::new(BatchMeta::new())),
-            client_rqs: vec![],
-        }
-    }
-
-    /// Update our log to reflect the new views
-    pub fn update_current_view(&mut self, view: &ViewInfo) {
-        self.leader_set = view.leader_set().clone();
-        self.request_space_slices = view.hash_space_division().clone();
-    }
-
-    /// Getter for batch_meta
-    pub fn batch_meta(&self) -> &Arc<Mutex<BatchMeta>> {
-        &self.batch_meta
-    }
-
-    /// Getter for the current digest
-    pub fn current_digest(&self) -> Option<Digest> {
-        self.current_digest.clone()
-    }
-
-    /// Get the current batch size in this consensus decision
-    pub fn current_batch_size(&self) -> usize { self.current_batch_size }
-
-    /// Inform the log that we are now processing a new batch of operations
-    pub fn process_pre_prepare(&mut self,
-                               request_batch: StoredConsensusMessage<O>,
-                               digest: Digest,
-                               mut batch_rq_digests: Vec<ClientRqInfo>) -> Result<Option<FullBatch>> {
-        let start = Instant::now();
-
-        let sending_leader = request_batch.header().from();
-
-        let slice = self.request_space_slices.get(&sending_leader)
-            .ok_or(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                          format!("Failed to get request space for leader {:?}. Len: {:?}. {:?}",
-                                                  sending_leader,
-                                                  self.request_space_slices.len(),
-                                                  self.request_space_slices).as_str()))?;
-
-        if request_batch.header().from() != self.node_id {
-            for request in &batch_rq_digests {
-                if !crate::bft::sync::view::is_request_in_hash_space(&request.digest(), slice) {
-                    return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                                      "This batch contains requests that are not in the hash space of the leader."));
-                }
-            }
-        }
-
-        self.duplicate_detection.insert_pre_prepare_received(sending_leader)?;
-
-        // Get the correct index for this batch
-        let leader_index = pre_prepare_index_of(&self.leader_set, &sending_leader)?;
-
-        self.ongoing_decision.insert_pre_prepare(leader_index, request_batch.clone());
-
-        self.current_received_pre_prepares += 1;
-
-        //
-        self.current_batch_size += batch_rq_digests.len();
-
-        self.client_rqs.append(&mut batch_rq_digests);
-
-        // Register this new batch as one that must be persisted for this batch to be executed
-        self.register_message_to_save(digest);
-
-        metric_duration(PRE_PREPARE_LOG_ANALYSIS_ID, start.elapsed());
-
-        // if we have received all of the messages in the set, calculate the digest.
-        Ok(if self.current_received_pre_prepares == self.leader_set.len() {
-            // We have received all of the required batches
-            let result = self.calculate_instance_digest();
-
-            let (digest, ordering) = result
-                .ok_or(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog, "Failed to calculate instance digest"))?;
-
-            self.current_digest = Some(digest.clone());
-
-            Some(ProofMetadata::new(self.seq_no, digest, ordering))
-        } else {
-            None
-        })
-    }
-
-    /// Calculate the instance of a completed consensus pre prepare phase with
-    /// all the batches received
-    fn calculate_instance_digest(&self) -> Option<(Digest, Vec<Digest>)> {
-        let mut ctx = Context::new();
-
-        let mut batch_ordered_digests = Vec::with_capacity(self.ongoing_decision.pre_prepare_digests.len());
-
-        for order_digest in &self.ongoing_decision.pre_prepare_digests {
-            if let Some(digest) = order_digest.clone() {
-                ctx.update(digest.as_ref());
-                batch_ordered_digests.push(digest);
-            } else {
-                return None;
-            }
-        }
-
-        Some((ctx.finish(), batch_ordered_digests))
-    }
-
-    /// Process the message received
-    pub(crate) fn process_message(&mut self, message: StoredConsensusMessage<O>) -> Result<()> {
-        match message.message().kind() {
-            ConsensusMessageKind::Prepare(_) => {
-                self.duplicate_detection.insert_prepare_received(message.header().from())?;
-            }
-            ConsensusMessageKind::Commit(_) => {
-                self.duplicate_detection.insert_commit_received(message.header().from())?;
-            }
-            _ => unreachable!()
-        }
-
-        self.ongoing_decision.insert_message(message);
-
-        Ok(())
-    }
-
-    /// Get the current decision
-    pub fn deciding(&self, f: usize) -> IncompleteProof {
-        let in_exec = self.seq_no;
-
-        self.ongoing_decision.deciding(in_exec, f)
-    }
-
-    /// Indicate that the batch is finished processing and
-    /// return the relevant information for it
-    pub fn finish_processing_batch(self) -> Option<CompletedBatch<O>> {
-        let new_meta = BatchMeta::new();
-        let batch_meta = std::mem::replace(&mut *self.batch_meta().lock().unwrap(), new_meta);
-
-        let OnGoingDecision {
-            pre_prepare_digests,
-            pre_prepare_messages,
-            prepare_messages,
-            commit_messages
-        } = self.ongoing_decision;
-
-        let current_digest = self.current_digest?;
-
-        let pre_prepare_ordering = pre_prepare_digests.into_iter().map(|elem| elem.unwrap()).collect();
-        let pre_prepare_messages = pre_prepare_messages.into_iter().map(|elem| elem.unwrap()).collect();
-
-        let messages_to_persist = self.current_messages_to_persist;
-
-        Some(CompletedBatch {
-            batch_digest: current_digest,
-            seq_no: self.seq_no,
-            pre_prepare_ordering,
-            pre_prepare_messages,
-            prepare_messages,
-            commit_messages,
-            client_requests: self.client_rqs,
-            messages_to_persist,
-            batch_meta,
-        })
-    }
-
-    fn register_message_to_save(&mut self, message: Digest) {
-        self.current_messages_to_persist.push(message);
-    }
-}
-
-impl<O> OnGoingDecision<O> {
-    pub fn initialize(leader_count: usize) -> Self {
-        Self {
-            pre_prepare_digests: iter::repeat(None).take(leader_count).collect(),
-            pre_prepare_messages: iter::repeat(None).take(leader_count).collect(),
-            prepare_messages: Vec::new(),
-            commit_messages: Vec::new(),
-        }
-    }
-
-    pub fn initialize_with_ordering(leader_count: usize, ordering: Vec<Digest>) -> Self {
-        Self {
-            pre_prepare_digests: ordering.into_iter().map(|d| Some(d)).collect(),
-            pre_prepare_messages: iter::repeat(None).take(leader_count).collect(),
-            prepare_messages: Vec::new(),
-            commit_messages: Vec::new(),
-        }
-    }
-
-    /// Insert a pre prepare message into this on going decision
-    fn insert_pre_prepare(&mut self, index: usize, pre_prepare: StoredConsensusMessage<O>) {
-        if index >= self.pre_prepare_messages.len() {
-            unreachable!("Cannot insert a pre prepare message that was sent by a leader that is out of bounds")
-        }
-
-        self.pre_prepare_digests[index] = Some(pre_prepare.header().digest().clone());
-        self.pre_prepare_messages[index] = Some(pre_prepare);
-    }
-
-    /// Insert a consensus message into this on going decision
-    fn insert_message(&mut self, message: StoredConsensusMessage<O>) {
-        match message.message().kind() {
-            ConsensusMessageKind::Prepare(_) => {
-                self.prepare_messages.push(message);
-            }
-            ConsensusMessageKind::Commit(_) => {
-                self.commit_messages.push(message);
-            }
-            _ => {
-                unreachable!("Please use insert_pre_prepare to insert a pre prepare message")
-            }
-        }
-    }
-
-    /// Insert a message from the stored message into this on going decision
-    pub fn insert_persisted_msg(&mut self, message: StoredConsensusMessage<O>) -> Result<()> {
-        match message.message().kind() {
-            ConsensusMessageKind::PrePrepare(_) => {
-                let index = pre_prepare_index_from_digest_opt(&self.pre_prepare_digests, message.header().digest())?;
-
-                self.pre_prepare_messages[index] = Some(message);
-            }
-            _ => {
-                self.insert_message(message);
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Get the current decision
-    pub fn deciding(&self, in_exec: SeqNo, f: usize) -> IncompleteProof {
-
-        // fetch write set
-        let write_set = PrepareSet({
-            let mut buf = Vec::new();
-
-            for stored in self.prepare_messages.iter().rev() {
-                match stored.message().sequence_number().cmp(&in_exec) {
-                    Ordering::Equal => {
-                        let digest = match stored.message().kind() {
-                            ConsensusMessageKind::Prepare(d) => d.clone(),
-                            _ => unreachable!(),
-                        };
-
-                        buf.push(ViewDecisionPair(
-                            stored.message().view(),
-                            digest,
-                        ));
-                    }
-                    Ordering::Less => break,
-                    // impossible, because we are executing `in_exec`
-                    Ordering::Greater => unreachable!(),
-                }
-            }
-
-            buf
-        });
-
-        // fetch quorum prepares
-        let quorum_prepares = 'outer: loop {
-            // NOTE: check `last_decision` comment on quorum
-            let quorum = f << 1;
-            let mut last_view = None;
-            let mut count = 0;
-
-            for stored in self.prepare_messages.iter().rev() {
-                match stored.message().sequence_number().cmp(&in_exec) {
-                    Ordering::Equal => {
-                        match last_view {
-                            None => (),
-                            Some(v) if stored.message().view() == v => (),
-                            _ => count = 0,
-                        }
-                        last_view = Some(stored.message().view());
-                        count += 1;
-                        if count == quorum {
-                            let digest = match stored.message().kind() {
-                                ConsensusMessageKind::Prepare(d) => d.clone(),
-                                _ => unreachable!(),
-                            };
-                            break 'outer Some(ViewDecisionPair(stored.message().view(), digest));
-                        }
-                    }
-                    Ordering::Less => break,
-                    // impossible, because we are executing `in_exec`
-                    Ordering::Greater => unreachable!(),
-                }
-            }
-
-            break 'outer None;
-        };
-
-        IncompleteProof::new(in_exec, write_set, quorum_prepares)
-    }
-}
-
-impl<O> Orderable for DecidingLog<O> {
-    fn sequence_number(&self) -> SeqNo {
-        self.seq_no
-    }
-}
-
-impl DuplicateReplicaEvaluator {
-    fn insert_pre_prepare_received(&mut self, node_id: NodeId) -> Result<()> {
-        if !self.received_pre_prepare_messages.insert(node_id) {
-            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                              "We have already received a message from that leader."));
-        }
-
-        Ok(())
-    }
-    fn insert_prepare_received(&mut self, node_id: NodeId) -> Result<()> {
-        if !self.received_prepare_messages.insert(node_id) {
-            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                              "We have already received a message from that leader."));
-        }
-
-        Ok(())
-    }
-    fn insert_commit_received(&mut self, node_id: NodeId) -> Result<()> {
-        if !self.received_commit_messages.insert(node_id) {
-            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                              "We have already received a message from that leader."));
-        }
-
-        Ok(())
-    }
-}
-
-impl<O> Orderable for CompletedBatch<O> {
-    fn sequence_number(&self) -> SeqNo {
-        self.seq_no
-    }
-}
-
-impl<O> CompletedBatch<O> {
-    pub fn batch_digest(&self) -> Digest {
-        self.batch_digest
-    }
-    pub fn pre_prepare_ordering(&self) -> &Vec<Digest> {
-        &self.pre_prepare_ordering
-    }
-    pub fn pre_prepare_messages(&self) -> &Vec<StoredConsensusMessage<O>> {
-        &self.pre_prepare_messages
-    }
-
-    pub fn messages_to_persist(&self) -> &Vec<Digest> {
-        &self.messages_to_persist
-    }
-    pub fn batch_meta(&self) -> &BatchMeta {
-        &self.batch_meta
-    }
-
-    pub fn request_count(&self) -> usize {
-        self.client_requests.len()
-    }
-
-    /// Create a proof from the completed batch
-    pub fn proof(&self, quorum: Option<usize>) -> Result<Proof<O>> {
-        let (leader_count, order) = (self.pre_prepare_ordering.len(), &self.pre_prepare_ordering);
-
-        if self.pre_prepare_messages.len() != leader_count {
-            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecisions,
-                                              format!("Failed to create a proof, pre_prepares do not match up to the leader count {} leader count {} preprepares",
-                                                      leader_count, self.prepare_messages.len()).as_str()));
-        }
-
-        let mut ordered_pre_prepares = Vec::with_capacity(leader_count);
-
-        for digest in order {
-            for i in 0..self.pre_prepare_messages.len() {
-                let message = &self.pre_prepare_messages[i];
-
-                if *message.header().digest() == *digest {
-                    ordered_pre_prepares.push(self.pre_prepare_messages[i].clone());
-
-                    break;
-                }
-            }
-        }
-
-        if let Some(quorum) = quorum {
-            if self.prepare_messages.len() < quorum {
-                return Err(Error::simple_with_msg(ErrorKind::MsgLogDecisions,
-                                                  "Failed to create a proof, prepares do not match up to the 2*f+1"));
-            }
-
-            if self.commit_messages.len() < quorum {
-                return Err(Error::simple_with_msg(ErrorKind::MsgLogDecisions,
-                                                  "Failed to create a proof, commits do not match up to the 2*f+1"));
-            }
-        }
-
-        let metadata = ProofMetadata::new(self.seq_no,
-                                          self.batch_digest.clone(),
-                                          order.clone());
-
-        Ok(Proof::new(metadata,
-                      ordered_pre_prepares,
-                      self.prepare_messages.clone(),
-                      self.commit_messages.clone()))
-    }
-}
-
-pub fn pre_prepare_index_from_digest_opt(prepare_set: &Vec<Option<Digest>>, digest: &Digest) -> Result<usize> {
-    match prepare_set.iter().position(|pre_prepare| pre_prepare.map(|d| d == *digest).unwrap_or(false)) {
-        None => {
-            Err(Error::simple_with_msg(ErrorKind::Consensus, "Pre prepare is not part of the pre prepare set"))
-        }
-        Some(pos) => {
-            Ok(pos)
-        }
-    }
-}
-
-pub fn pre_prepare_index_of_from_digest(prepare_set: &Vec<Digest>, preprepare: &Digest) -> Result<usize> {
-    match prepare_set.iter().position(|pre_prepare| *pre_prepare == *preprepare) {
-        None => {
-            Err(Error::simple_with_msg(ErrorKind::Consensus, "Pre prepare is not part of the pre prepare set"))
-        }
-        Some(pos) => {
-            Ok(pos)
-        }
-    }
-}
-
-pub fn pre_prepare_index_of(leader_set: &Vec<NodeId>, proposer: &NodeId) -> Result<usize> {
-    match leader_set.iter().position(|node| *node == *proposer) {
-        None => {
-            Err(Error::simple_with_msg(ErrorKind::Consensus, "Proposer is not part of the leader set"))
-        }
-        Some(pos) => {
-            Ok(pos)
-        }
-    }
-}
-
-pub fn make_proof_from<O>(proof_meta: ProofMetadata, mut ongoing: OnGoingDecision<O>) -> Result<Proof<O>> {
-    let OnGoingDecision {
-        pre_prepare_digests,
-        pre_prepare_messages,
-        prepare_messages,
-        commit_messages
-    } = ongoing;
-
-    let pre_prepare_messages: Vec<StoredConsensusMessage<O>> = pre_prepare_messages.into_iter()
-        .map(|elem| {
-            elem.unwrap()
-        }).collect();
-
-    for (digest, digest2) in zip(proof_meta.pre_prepare_ordering(), &pre_prepare_digests) {
-        let digest2 = digest2.as_ref().ok_or(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog, "Failed to create a proof, pre prepare messages are missing"))?;
-
-        if digest != digest2 {
-            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                              "Failed to create a proof, pre prepares do not match up to the ordering"));
-        }
-    }
-
-    Ok(Proof::new(proof_meta, pre_prepare_messages, prepare_messages, commit_messages))
-}
-
-impl<O> From<CompletedBatch<O>> for DecisionInformation {
-    fn from(value: CompletedBatch<O>) -> Self {
-        DecisionInformation::new(value.batch_digest,
-                                 value.messages_to_persist,
-                                 value.client_requests)
-    }
-}
\ No newline at end of file

From d8efe8b881b565959bdb9acde9d20fd5b22a0fe2 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Sat, 7 Oct 2023 15:55:51 +0100
Subject: [PATCH 57/80] More work on the decision log, start removing the log
 dependency in most places. Made the shareable message type the default in
 order to facilitate message sharing without cloning.

---
 .../src/bft/consensus/decision/mod.rs         | 69 ++++++++--------
 febft-pbft-consensus/src/bft/consensus/mod.rs | 82 ++++++++++---------
 .../src/bft/log/deciding/mod.rs               |  5 +-
 febft-pbft-consensus/src/bft/log/mod.rs       | 14 ++++
 febft-pbft-consensus/src/bft/mod.rs           | 64 +++++++--------
 febft-pbft-consensus/src/bft/sync/mod.rs      | 42 ++++------
 6 files changed, 141 insertions(+), 135 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 2dedbfb2..7e3fcf35 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -13,6 +13,7 @@ use atlas_communication::message::{Header, StoredMessage};
 use atlas_core::messages::ClientRqInfo;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog};
+use atlas_core::smr::smr_decision_log::ShareableMessage;
 use atlas_core::timeouts::Timeouts;
 use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_duration;
@@ -21,7 +22,7 @@ use crate::bft::consensus::accessory::{AccessoryConsensus, ConsensusDecisionAcce
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
 use crate::bft::log::deciding::{CompletedBatch, WorkingDecisionLog};
 use crate::bft::log::decisions::IncompleteProof;
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{ConsensusMetrics, PRE_PREPARE_ANALYSIS_ID};
 use crate::bft::PBFT;
@@ -82,25 +83,25 @@ pub enum DecisionStatus<O> {
     MessageQueued,
     /// A `febft` quorum still hasn't made a decision
     /// on a client request to be executed.
-    Deciding(StoredMessage<ConsensusMessage<O>>),
+    Deciding(ShareableMessage<PBFTMessage<O>>),
     /// Transitioned to another next phase of the consensus decision
-    Transitioned(StoredMessage<ConsensusMessage<O>>),
+    Transitioned(ShareableMessage<PBFTMessage<O>>),
     /// A `febft` quorum decided on the execution of
     /// the batch of requests with the given digests.
     /// The first digest is the digest of the Prepare message
     /// And therefore the entire batch digest
     /// THe second Vec<Digest> is a vec with digests of the requests contained in the batch
     /// The third is the messages that should be persisted for this batch to be considered persisted
-    Decided(StoredMessage<ConsensusMessage<O>>),
+    Decided(ShareableMessage<PBFTMessage<O>>),
     DecidedIgnored,
 }
 
 /// A message queue for this particular consensus instance
 pub struct MessageQueue<O> {
     get_queue: bool,
-    pre_prepares: VecDeque<StoredMessage<ConsensusMessage<O>>>,
-    prepares: VecDeque<StoredMessage<ConsensusMessage<O>>>,
-    commits: VecDeque<StoredMessage<ConsensusMessage<O>>>,
+    pre_prepares: VecDeque<ShareableMessage<PBFTMessage<O>>>,
+    prepares: VecDeque<ShareableMessage<PBFTMessage<O>>>,
+    commits: VecDeque<ShareableMessage<PBFTMessage<O>>>,
 }
 
 /// The information needed to make a decision on a batch of requests.
@@ -133,9 +134,9 @@ impl<O> MessageQueue<O> {
         }
     }
 
-    pub(super) fn from_messages(pre_prepares: VecDeque<StoredMessage<ConsensusMessage<O>>>,
-                                prepares: VecDeque<StoredMessage<ConsensusMessage<O>>>,
-                                commits: VecDeque<StoredMessage<ConsensusMessage<O>>>) -> Self {
+    pub(super) fn from_messages(pre_prepares: VecDeque<ShareableMessage<PBFTMessage<O>>>,
+                                prepares: VecDeque<ShareableMessage<PBFTMessage<O>>>,
+                                commits: VecDeque<ShareableMessage<PBFTMessage<O>>>) -> Self {
         let get_queue = !pre_prepares.is_empty() || !prepares.is_empty() || !commits.is_empty();
 
         Self {
@@ -152,19 +153,19 @@ impl<O> MessageQueue<O> {
 
     pub fn is_signalled(&self) -> bool { self.get_queue }
 
-    fn queue_pre_prepare(&mut self, message: StoredMessage<ConsensusMessage<O>>) {
+    fn queue_pre_prepare(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
         self.pre_prepares.push_back(message);
 
         self.signal();
     }
 
-    fn queue_prepare(&mut self, message: StoredMessage<ConsensusMessage<O>>) {
+    fn queue_prepare(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
         self.prepares.push_back(message);
 
         self.signal();
     }
 
-    fn queue_commit(&mut self, message: StoredMessage<ConsensusMessage<O>>) {
+    fn queue_commit(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
         self.commits.push_back(message);
 
         self.signal();
@@ -199,16 +200,16 @@ impl<D, PL> ConsensusDecision<D, PL>
         }
     }
 
-    pub fn queue(&mut self, header: Header, message: ConsensusMessage<D::Request>) {
+    pub fn queue(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) {
         match message.kind() {
             ConsensusMessageKind::PrePrepare(_) => {
-                self.message_queue.queue_pre_prepare(StoredMessage::new(header, message));
+                self.message_queue.queue_pre_prepare(message);
             }
             ConsensusMessageKind::Prepare(_) => {
-                self.message_queue.queue_prepare(StoredMessage::new(header, message));
+                self.message_queue.queue_prepare(message);
             }
             ConsensusMessageKind::Commit(_) => {
-                self.message_queue.queue_commit(StoredMessage::new(header, message));
+                self.message_queue.queue_commit(message);
             }
         }
     }
@@ -256,15 +257,15 @@ impl<D, PL> ConsensusDecision<D, PL>
 
     /// Process a message relating to this consensus instance
     pub fn process_message<NT>(&mut self,
-                               header: Header,
-                               message: ConsensusMessage<D::Request>,
+                               s_message: ShareableMessage<PBFTMessage<D::Request>>,
                                synchronizer: &Synchronizer<D>,
                                timeouts: &Timeouts,
-                               log: &mut Log<D, PL>,
                                node: &Arc<NT>) -> Result<DecisionStatus<D::Request>>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
               PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let view = synchronizer.view();
+        let header = s_message.header();
+        let message = s_message.message().consensus();
 
         return match self.phase {
             DecisionPhase::Initialize => {
@@ -272,7 +273,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                 // This consensus instance.
                 warn!("{:?} // Queueing message {:?} as we are in the initialize phase", self.node_id, message);
 
-                self.queue(header, message);
+                self.queue(s_message);
 
                 return Ok(DecisionStatus::MessageQueued);
             }
@@ -306,7 +307,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         debug!("{:?} // Received {:?} from {:?} while in prepreparing ",
                             self.node_id, message, header.from());
 
-                        self.message_queue.queue_prepare(StoredMessage::new(header, message));
+                        self.message_queue.queue_prepare(s_message);
 
                         return Ok(DecisionStatus::MessageQueued);
                     }
@@ -314,7 +315,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         debug!("{:?} // Received {:?} from {:?} while in pre preparing",
                             self.node_id, message, header.from());
 
-                        self.message_queue.queue_commit(StoredMessage::new(header, message));
+                        self.message_queue.queue_commit(s_message);
 
                         return Ok(DecisionStatus::MessageQueued);
                     }
@@ -332,13 +333,13 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                 //TODO: Try out cloning each request on this method,
                 let mut digests = request_batch_received(
-                    &message,
+                    message,
                     timeouts,
                     synchronizer,
                     &mut self.working_log,
                 );
 
-                let batch_metadata = self.working_log.process_pre_prepare(header.clone(), &message,
+                let batch_metadata = self.working_log.process_pre_prepare(header.clone(), message,
                                                                           header.digest().clone(), digests)?;
 
                 let mut result;
@@ -363,17 +364,13 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                     let current_digest = batch_metadata.batch_digest();
 
-                    // Register that all of the batches have been received
-                    // The digest of the batch and the order of the batches
-                    log.all_batches_received(batch_metadata);
-
                     self.accessory.handle_pre_prepare_phase_completed(&self.working_log,
                                                                       &view, &header, &message, node);
 
                     self.message_queue.signal();
 
                     // Mark that we have transitioned to the next phase
-                    result = DecisionStatus::Transitioned(StoredMessage::new(header, message));
+                    result = DecisionStatus::Transitioned(s_message);
 
                     // We no longer start the count at 1 since all leaders must also send the prepare
                     // message with the digest of the entire batch
@@ -385,7 +382,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.accessory.handle_partial_pre_prepare(&self.working_log, &view,
                                                               &header, &message, &**node);
 
-                    result = DecisionStatus::Deciding(StoredMessage::new(header, message));
+                    result = DecisionStatus::Deciding(s_message);
 
                     DecisionPhase::PrePreparing(received)
                 };
@@ -405,7 +402,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         debug!("{:?} // Received {:?} from {:?} while in preparing phase",
                             self.node_id, message, header.from());
 
-                        self.message_queue.queue_commit(StoredMessage::new(header, message));
+                        self.message_queue.queue_commit(s_message);
 
                         return Ok(DecisionStatus::MessageQueued);
                     }
@@ -458,7 +455,7 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                     self.message_queue.signal();
 
-                    result = DecisionStatus::Transitioned(StoredMessage::new(header, message));
+                    result = DecisionStatus::Transitioned(s_message);
 
                     DecisionPhase::Committing(0)
                 } else {
@@ -468,7 +465,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.accessory.handle_preparing_no_quorum(&self.working_log, &view,
                                                               &header, &message, &**node);
 
-                    result = DecisionStatus::Deciding(StoredMessage::new(header, message));
+                    result = DecisionStatus::Deciding(s_message);
 
                     DecisionPhase::Preparing(received)
                 };
@@ -527,7 +524,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.accessory.handle_committing_quorum(&self.working_log, &view,
                                                             &header, &message, &**node);
 
-                    Ok(DecisionStatus::Decided(StoredMessage::new(header, message)))
+                    Ok(DecisionStatus::Decided(s_message))
                 } else {
                     debug!("{:?} // Received commit message {:?} from {:?}. Current count {}",
                         self.node_id, stored_msg.message().sequence_number(), header.from(), received);
@@ -537,7 +534,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.accessory.handle_committing_no_quorum(&self.working_log, &view,
                                                                &header, &message, &**node);
 
-                    Ok(DecisionStatus::Deciding(StoredMessage::new(header, message)))
+                    Ok(DecisionStatus::Deciding(s_message))
                 };
             }
             DecisionPhase::Decided => {
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index b55e3878..5dc35d79 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -8,6 +8,7 @@ use event_listener::Event;
 use log::{debug, error, info, trace, warn};
 
 use atlas_common::error::*;
+use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_advance_message_queue_return, tbo_queue_message};
 use atlas_communication::message::{Header, StoredMessage};
@@ -16,6 +17,7 @@ use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::persistent_log::{OrderingProtocolLog};
+use atlas_core::smr::smr_decision_log::ShareableMessage;
 use atlas_core::timeouts::Timeouts;
 use atlas_smr_application::ExecutorHandle;
 use atlas_smr_application::serialize::ApplicationData;
@@ -24,7 +26,8 @@ use atlas_metrics::metrics::metric_increment;
 use crate::bft::{PBFT, SysMsg};
 use crate::bft::consensus::decision::{ConsensusDecision, DecisionPollStatus, DecisionStatus, MessageQueue};
 use crate::bft::log::deciding::CompletedBatch;
-use crate::bft::log::decisions::IncompleteProof;
+use crate::bft::log::decisions::{DecisionLog, IncompleteProof, Proof};
+use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::OPERATIONS_PROCESSED_ID;
@@ -78,9 +81,9 @@ pub struct TboQueue<O> {
     curr_seq: SeqNo,
     watermark: u32,
     get_queue: bool,
-    pre_prepares: VecDeque<VecDeque<StoredMessage<ConsensusMessage<O>>>>,
-    prepares: VecDeque<VecDeque<StoredMessage<ConsensusMessage<O>>>>,
-    commits: VecDeque<VecDeque<StoredMessage<ConsensusMessage<O>>>>,
+    pre_prepares: VecDeque<VecDeque<ShareableMessage<PBFTMessage<O>>>>,
+    prepares: VecDeque<VecDeque<ShareableMessage<PBFTMessage<O>>>>,
+    commits: VecDeque<VecDeque<ShareableMessage<PBFTMessage<O>>>>,
 }
 
 impl<O> Orderable for TboQueue<O> {
@@ -136,34 +139,34 @@ impl<O> TboQueue<O> {
 
     /// Queues a consensus message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
-    pub fn queue(&mut self, h: Header, m: ConsensusMessage<O>) {
-        match m.kind() {
-            ConsensusMessageKind::PrePrepare(_) => self.queue_pre_prepare(h, m),
-            ConsensusMessageKind::Prepare(_) => self.queue_prepare(h, m),
-            ConsensusMessageKind::Commit(_) => self.queue_commit(h, m),
+    pub fn queue(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
+        match message.message().consensus().kind() {
+            ConsensusMessageKind::PrePrepare(_) => self.queue_pre_prepare(message),
+            ConsensusMessageKind::Prepare(_) => self.queue_prepare(message),
+            ConsensusMessageKind::Commit(_) => self.queue_commit(message),
         }
     }
 
     /// Queues a `PRE-PREPARE` message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
-    fn queue_pre_prepare(&mut self, h: Header, m: ConsensusMessage<O>) {
+    fn queue_pre_prepare(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
         tbo_queue_message(
             self.base_seq(),
             &mut self.pre_prepares,
-            StoredMessage::new(h, m),
+            message,
         )
     }
 
     /// Queues a `PREPARE` message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
-    fn queue_prepare(&mut self, h: Header, m: ConsensusMessage<O>) {
-        tbo_queue_message(self.base_seq(), &mut self.prepares, StoredMessage::new(h, m))
+    fn queue_prepare(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
+        tbo_queue_message(self.base_seq(), &mut self.prepares, message)
     }
 
     /// Queues a `COMMIT` message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
-    fn queue_commit(&mut self, h: Header, m: ConsensusMessage<O>) {
-        tbo_queue_message(self.base_seq(), &mut self.commits, StoredMessage::new(h, m))
+    fn queue_commit(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
+        tbo_queue_message(self.base_seq(), &mut self.commits, message)
     }
 
     /// Clear this queue
@@ -211,7 +214,7 @@ pub struct Consensus<D, PL>
     tbo_queue: TboQueue<D::Request>,
     /// This queue serves for us to keep track of messages we receive of coming up views.
     /// This is important for us to be able to continue the process of moving views after a view change
-    view_queue: VecDeque<Vec<StoredMessage<ConsensusMessage<D::Request>>>>,
+    view_queue: VecDeque<Vec<ShareableMessage<PBFTMessage<D::Request>>>>,
     /// The consensus guard that will be used to ensure that the proposer only proposes one batch
     /// for each consensus instance
     consensus_guard: Arc<ProposerConsensusGuard>,
@@ -264,14 +267,16 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
     }
 
     /// Queue a given message into our message queues.
-    pub fn queue(&mut self, header: Header, message: ConsensusMessage<D::Request>) {
-        let message_seq = message.sequence_number();
+    pub fn queue(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) {
+        let message_seq = message.message().sequence_number();
 
-        let view_seq = message.view();
+        let view_seq = message.message().consensus().view();
+
+        let header = message.header();
 
         match view_seq.index(self.curr_view.sequence_number()) {
             Either::Right(i) if i > 0 => {
-                self.enqueue_other_view_message(i, header, message);
+                self.enqueue_other_view_message(i, message);
 
                 return;
             }
@@ -303,12 +308,13 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
 
             // We are not currently processing this consensus instance
             // so we need to queue the message
-            self.tbo_queue.queue(header, message);
+            self.tbo_queue.queue(message);
         } else {
             debug!("{:?} // Queueing message out of context msg {:?} received from {:?} into the corresponding decision {}",
                 self.node_id, message, header.from(), i);
+
             // Queue the message in the corresponding pending decision
-            self.decisions.get_mut(i).unwrap().queue(header, message);
+            self.decisions.get_mut(i).unwrap().queue(message);
 
             // Signal that we are ready to receive messages
             self.signalled.push_signalled(message_seq);
@@ -363,21 +369,21 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
     }
 
     pub fn process_message<NT>(&mut self,
-                               header: Header,
-                               message: ConsensusMessage<D::Request>,
+                               s_message: ShareableMessage<PBFTMessage<D::Request>>,
                                synchronizer: &Synchronizer<D>,
                                timeouts: &Timeouts,
-                               log: &mut Log<D, PL>,
                                node: &Arc<NT>) -> Result<ConsensusStatus>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
               PL: OrderingProtocolLog<D, PBFTConsensus<D>>, {
+        let (header, message) = (s_message.header(), s_message.message().consensus());
+
         let message_seq = message.sequence_number();
 
         let view_seq = message.view();
 
         match view_seq.index(self.curr_view.sequence_number()) {
             Either::Right(i) if i > 0 => {
-                self.enqueue_other_view_message(i, header, message);
+                self.enqueue_other_view_message(i, s_message);
 
                 return Ok(ConsensusStatus::MessageQueued);
             }
@@ -405,7 +411,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             // so we need to queue the message
             debug!("{:?} // Queueing message {:?} for seq no {:?}", self.node_id, message, message_seq);
 
-            self.tbo_queue.queue(header, message);
+            self.tbo_queue.queue(s_message);
 
             return Ok(ConsensusStatus::MessageQueued);
         }
@@ -413,7 +419,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
         // Get the correct consensus instance for this message
         let decision = self.decisions.get_mut(i).unwrap();
 
-        let status = decision.process_message(header, message, synchronizer, timeouts, log, node)?;
+        let status = decision.process_message(s_message, synchronizer, timeouts,  node)?;
 
         Ok(match status {
             DecisionStatus::VotedTwice(node) => {
@@ -709,7 +715,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
                               seq: SeqNo,
                               view: &ViewInfo,
                               proof: Proof<D::Request>,
-                              log: &mut Log<D, PL>) -> Result<ProtocolConsensusDecision<D::Request>>
+                              log: &mut Log<D>) -> Result<ProtocolConsensusDecision<D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
 
         // If this is successful, it means that we are all caught up and can now start executing the
@@ -783,15 +789,13 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
 
         if let Some(messages) = option {
             for message in messages {
-                let (header, message) = message.into_inner();
-
-                self.queue(header, message);
+                self.queue(message);
             }
         }
     }
 
     /// Enqueue a message from another view into it's correct queue
-    fn enqueue_other_view_message(&mut self, index: usize, header: Header, message: ConsensusMessage<D::Request>) {
+    fn enqueue_other_view_message(&mut self, index: usize, message: ShareableMessage<PBFTMessage<D::Request>>) {
         debug!("{:?} // Enqueuing a message from another view into the view queue. Index {}  {:?}", self.node_id, index, message);
 
         // Adjust the index to be 0 based
@@ -801,7 +805,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             self.view_queue.push_back(Vec::new());
         }
 
-        self.view_queue[index].push(StoredMessage::new(header, message));
+        self.view_queue[index].push(message);
     }
 
 
@@ -812,9 +816,9 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
         new_view: &ViewInfo,
         synchronizer: &Synchronizer<D>,
         timeouts: &Timeouts,
-        log: &mut Log<D, PL>,
+        log: &mut Log<D>,
         node: &Arc<NT>,
-    ) where
+    ) -> Result<ConsensusStatus> where
         NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
         PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         //Prepare the algorithm as we are already entering this phase
@@ -837,9 +841,13 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
         // So the proposer won't try to propose anything to this decision
         self.decisions[0].skip_init_phase();
 
-        self.process_message(header, message, synchronizer, timeouts, log, node).unwrap();
+        let shareable_message = Arc::new(ReadOnly::new(StoredMessage::new(header, PBFTMessage::Consensus(message))));
+
+        let result = self.process_message(shareable_message, synchronizer, timeouts, node)?;
 
         self.consensus_guard.unlock_consensus();
+
+        Ok(result)
     }
 
     /// Collect the incomplete proof that is currently being decided
diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
index 23e4f2e0..46b31684 100644
--- a/febft-pbft-consensus/src/bft/log/deciding/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -10,10 +10,9 @@ use atlas_communication::message::Header;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
 use atlas_metrics::benchmarks::BatchMeta;
 use atlas_metrics::metrics::metric_duration;
+use crate::bft::log::decisions::{IncompleteProof, ProofMetadata};
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
 use crate::bft::metric::PRE_PREPARE_LOG_ANALYSIS_ID;
-use crate::bft::msg_log::deciding_log::{DecidingLog, FullBatch};
-use crate::bft::msg_log::decisions::{IncompleteProof, ProofMetadata};
 use crate::bft::sync::view::ViewInfo;
 
 /// Information about the completed batch, the contained requests and
@@ -103,7 +102,7 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
                                header: Header,
                                message: &ConsensusMessage<O>,
                                digest: Digest,
-                               mut batch_rq_digests: Vec<ClientRqInfo>) -> Result<Option<FullBatch>> {
+                               mut batch_rq_digests: Vec<ClientRqInfo>) -> Result<Option<ProofMetadata>> {
         let start = Instant::now();
 
         let sending_leader = header.from();
diff --git a/febft-pbft-consensus/src/bft/log/mod.rs b/febft-pbft-consensus/src/bft/log/mod.rs
index b417e083..f1916a80 100644
--- a/febft-pbft-consensus/src/bft/log/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/mod.rs
@@ -2,11 +2,25 @@ use atlas_common::node_id::NodeId;
 use atlas_common::ordering::SeqNo;
 use atlas_communication::message::Header;
 use atlas_core::messages::RequestMessage;
+use atlas_smr_application::serialize::ApplicationData;
+use crate::bft::log::decisions::DecisionLog;
 
 pub mod decided;
 pub mod deciding;
 pub mod decisions;
 
+pub struct Log<D> where D: ApplicationData {
+    decided: DecisionLog<D::Request>,
+}
+
+impl<D> Log<D> where D: ApplicationData {
+
+    pub fn decision_log(&self) -> &DecisionLog<D::Request> {
+        &self.decided
+    }
+
+}
+
 #[inline]
 pub fn operation_key<O>(header: &Header, message: &RequestMessage<O>) -> u64 {
     operation_key_raw(header.from(), message.session_id())
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 7bfb0e11..3e0550b0 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -10,8 +10,6 @@ use std::sync::atomic::AtomicBool;
 use std::time::Instant;
 use ::log::{debug, info, trace, warn};
 
-use log::{debug, info, trace, warn};
-
 use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
@@ -20,7 +18,7 @@ use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::serialize::Serializable;
 use atlas_core::messages::Protocol;
-use atlas_core::ordering_protocol::{ OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, PermissionedOrderingProtocol, ProtocolConsensusDecision, SerProof, SerProofMetadata, View};
+use atlas_core::ordering_protocol::{OPExecResult, OPPollResult, OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, PermissionedOrderingProtocol, ProtocolConsensusDecision, SerProof, SerProofMetadata, View};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::networking::serialize::{NetworkView, OrderingProtocolMessage};
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::{ReconfigurableOrderProtocol, ReconfigurationAttemptResult};
@@ -28,6 +26,7 @@ use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol,
 use atlas_core::reconfiguration_protocol::ReconfigurationProtocol;
 use atlas_core::request_pre_processing::RequestPreProcessor;
 use atlas_core::serialize::ReconfigurationProtocolMessage;
+use atlas_core::smr::smr_decision_log::ShareableMessage;
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_smr_application::ExecutorHandle;
 use atlas_smr_application::serialize::ApplicationData;
@@ -35,19 +34,16 @@ use atlas_metrics::metrics::metric_duration;
 
 use crate::bft::config::PBFTConfig;
 use crate::bft::consensus::{Consensus, ConsensusPollStatus, ConsensusStatus, ProposerConsensusGuard};
+use crate::bft::log::decisions::ProofMetadata;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, ObserveEventKind, PBFTMessage, ViewChangeMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{CONSENSUS_INSTALL_STATE_TIME_ID, MSG_LOG_INSTALL_TIME_ID};
-use crate::bft::msg_log::initialize_decided_log;
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::{DecisionLog, Proof};
 use crate::bft::proposer::Proposer;
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer, SynchronizerPollStatus, SynchronizerStatus, SyncReconfigurationResult};
 
 pub mod consensus;
 pub mod proposer;
 pub mod sync;
-pub mod msg_log;
 pub mod log;
 pub mod config;
 pub mod message;
@@ -143,10 +139,8 @@ impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
     }
 
 
-    fn handle_off_ctx_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>)
+    fn handle_off_ctx_message(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>)
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        let (header, message) = message.into_inner();
-
         match message.into_inner() {
             PBFTMessage::Consensus(consensus) => {
                 debug!("{:?} // Received off context consensus message {:?}", self.node.id(), consensus);
@@ -179,7 +173,7 @@ impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
         Ok(())
     }
 
-    fn poll(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
+    fn poll(&mut self) -> OPPollResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         trace!("{:?} // Polling {:?}", self.node.id(), self.phase);
 
@@ -193,7 +187,7 @@ impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
         }
     }
 
-    fn process_message(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OrderProtocolExecResult<D::Request>>
+    fn process_message(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         match self.phase {
             ConsensusPhase::NormalPhase => {
@@ -205,31 +199,18 @@ impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
         }
     }
 
-    fn sequence_number_with_proof(&self) -> Result<Option<(SeqNo, SerProof<D, Self::Serialization>)>> {
-        Ok(self.message_log.last_proof(self.synchronizer.view().f())
-            .map(|p| (p.sequence_number(), p)))
-    }
-
-    fn verify_sequence_number(&self, seq_no: SeqNo, proof: &SerProof<D, Self::Serialization>) -> Result<bool> {
-        let proof: &Proof<D::Request> = proof;
-
-        //TODO: Verify the proof
-
-        Ok(true)
-    }
-
     fn install_seq_no(&mut self, seq_no: SeqNo) -> Result<()> {
         self.consensus.install_sequence_number(seq_no, &self.synchronizer.view());
 
         Ok(())
     }
 
-    fn handle_timeout(&mut self, timeout: Vec<RqTimeout>) -> Result<OrderProtocolExecResult<D::Request>>
+    fn handle_timeout(&mut self, timeout: Vec<RqTimeout>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         if self.consensus.is_catching_up() {
             warn!("{:?} // Ignoring timeouts while catching up", self.node.id());
 
-            return Ok(OrderProtocolExecResult::Success);
+            return Ok(OPExecResult::MessageDropped);
         }
 
         let status = self.synchronizer.client_requests_timed_out(self.node.id(), &timeout);
@@ -260,7 +241,7 @@ impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
             _ => (),
         }
 
-        Ok(OrderProtocolExecResult::Success)
+        Ok(OPExecResult::MessageProcessedNoUpdate)
     }
 }
 
@@ -268,7 +249,6 @@ impl<D, NT, PL> PermissionedOrderingProtocol for PBFTOrderProtocol<D, NT, PL>
     where D: ApplicationData + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
           PL: Clone {
-
     type PermissionedSerialization = PBFTConsensus<D>;
 
     fn view(&self) -> View<Self::PermissionedSerialization> {
@@ -297,7 +277,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
 
         let OrderingProtocolArgs(executor, timeouts,
                                  pre_processor, batch_input,
-                                 node,  quorum) = args;
+                                 node, quorum) = args;
 
         let sync = Synchronizer::initialize_with_quorum(node_id, view.sequence_number(), quorum.clone(), timeout_dur)?;
 
@@ -535,9 +515,8 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     /// Advance the consensus phase with a received message
     fn adv_consensus(
         &mut self,
-        header: Header,
-        message: ConsensusMessage<D::Request>,
-    ) -> Result<OrderProtocolExecResult<D::Request>>
+        message: ShareableMessage<PBFTMessage<D::Request>>,
+    ) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let seq = self.consensus.sequence_number();
 
@@ -548,14 +527,29 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         // );
 
         let status = self.consensus.process_message(
-            header,
             message,
             &self.synchronizer,
             &self.timeouts,
-            &mut self.message_log,
             &self.node,
         )?;
 
+        match status {
+            ConsensusStatus::VotedTwice(_) | ConsensusStatus::MessageIgnored => {
+                OPExecResult::MessageDropped
+            }
+            ConsensusStatus::MessageQueued => {
+                OPExecResult::MessageQueued
+            }
+            ConsensusStatus::Deciding => {
+
+            }
+            ConsensusStatus::Decided => {
+                let finalized_decisions = self.finalize_all_possible()?;
+
+                return Ok(OPExecResult::Decided())
+            }
+        }
+
         match status {
             // if deciding, nothing to do
             ConsensusStatus::Deciding => {}
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index b6cfb98a..b7a55e32 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -26,16 +26,16 @@ use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::ProtocolConsensusDecision;
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::ReconfigurationAttemptResult;
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
+use atlas_core::persistent_log::{OrderingProtocolLog};
 use atlas_core::request_pre_processing::RequestPreProcessor;
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_smr_application::serialize::ApplicationData;
 
-use crate::bft::consensus::Consensus;
+use crate::bft::consensus::{Consensus, ConsensusStatus};
+use crate::bft::log::decisions::{CollectData, Proof, ViewDecisionPair};
+use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, FwdConsensusMessage, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::{CollectData, Proof, StoredConsensusMessage, ViewDecisionPair};
 use crate::bft::PBFT;
 use crate::bft::sync::view::ViewInfo;
 
@@ -121,7 +121,6 @@ pub struct LeaderCollects<O> {
 }
 
 impl<O> LeaderCollects<O> {
-
     pub fn message(&self) -> &FwdConsensusMessage<O> {
         &self.proposed
     }
@@ -368,10 +367,10 @@ pub enum SynchronizerStatus<O> {
     /// The view change protocol is currently running.
     Running,
     /// The view change protocol just finished running.
-    NewView(Option<ProtocolConsensusDecision<O>>),
+    NewView(ConsensusStatus, Option<ProtocolConsensusDecision<O>>),
     /// The view change protocol just finished running and we
     /// have successfully joined the quorum.
-    NewViewJoinedQuorum(Option<ProtocolConsensusDecision<O>>, NodeId),
+    NewViewJoinedQuorum(ConsensusStatus, Option<ProtocolConsensusDecision<O>>, NodeId),
 
     /// Before we finish the view change protocol, we need
     /// to run the CST protocol.
@@ -686,7 +685,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         header: Header,
         message: ViewChangeMessage<D::Request>,
         timeouts: &Timeouts,
-        log: &mut Log<D, PL>,
+        log: &mut Log<D>,
         rq_pre_processor: &RequestPreProcessor<D::Request>,
         consensus: &mut Consensus<D, PL>,
         node: &Arc<NT>,
@@ -1326,7 +1325,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     /// Resume the view change protocol after running the CST protocol.
     pub fn resume_view_change<NT, PL>(
         &self,
-        log: &mut Log<D, PL>,
+        log: &mut Log<D>,
         timeouts: &Timeouts,
         consensus: &mut Consensus<D, PL>,
         node: &Arc<NT>,
@@ -1334,7 +1333,6 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
               PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
-
         let state = self.finalize_state.borrow_mut().take()?;
 
         //This is kept alive until it is out of the scope
@@ -1354,7 +1352,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
     /// Start the quorum join procedure to integrate the given joining node into the current quorum
     /// of the system
-    pub fn start_join_quorum<NT, PL>(&self, joining_node: NodeId, node: &NT, timeouts: &Timeouts, log: &Log<D, PL>) -> SyncReconfigurationResult
+    pub fn start_join_quorum<NT, PL>(&self, joining_node: NodeId, node: &NT, timeouts: &Timeouts, log: &Log<D>) -> SyncReconfigurationResult
         where NT: OrderProtocolSendNode<D, PBFT<D>>,
               PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let current_view = self.view();
@@ -1379,11 +1377,11 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                     SyncReconfigurationResult::OnGoingQuorumChange(currently_adding)
                 } else {
                     SyncReconfigurationResult::OnGoingViewChange
-                }
+                };
             }
             ProtoPhase::ViewStopping2(_) | ProtoPhase::Stopping(_) | ProtoPhase::Stopping2(_) => {
                 // Here we still don't know what is the target of the view change.
-                return SyncReconfigurationResult::OnGoingViewChange
+                return SyncReconfigurationResult::OnGoingViewChange;
             }
             _ => {
                 info!("{:?} // Attempted to add node {:?} quorum but we are currently performing a view change", node.id(), joining_node);
@@ -1451,7 +1449,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                                             join_cert: Option<NodeId>,
                                             node: &NT,
                                             timeouts: &Timeouts,
-                                            _log: &Log<D, PL>, )
+                                            _log: &Log<D>, )
         where NT: OrderProtocolSendNode<D, PBFT<D>>,
               PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
@@ -1507,7 +1505,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         timed_out: Option<Vec<StoredRequestMessage<D::Request>>>,
         node: &NT,
         timeouts: &Timeouts,
-        _log: &Log<D, PL>,
+        _log: &Log<D>,
     ) where NT: OrderProtocolSendNode<D, PBFT<D>>,
             PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
@@ -1576,7 +1574,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         state: FinalizeState<D::Request>,
         proof: Option<&Proof<D::Request>>,
         _normalized_collects: Vec<Option<&CollectData<D::Request>>>,
-        log: &Log<D, PL>,
+        log: &Log<D>,
     ) -> FinalizeStatus<D::Request>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
@@ -1607,7 +1605,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     fn finalize<NT, PL>(
         &self,
         state: FinalizeState<D::Request>,
-        log: &mut Log<D, PL>,
+        log: &mut Log<D>,
         timeouts: &Timeouts,
         consensus: &mut Consensus<D, PL>,
         node: &Arc<NT>,
@@ -1629,10 +1627,6 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
         warn!("{:?} // Finalizing view change to view {:?} and consensus ID {:?}, Adding node? {:?}", node.id(), view, curr_cid, self.currently_adding_node.get());
 
-        // we will get some value to be proposed because of the
-        // check we did in `pre_finalize()`, guarding against no values
-        log.clear_last_occurrence(curr_cid);
-
         let (header, message) = proposed.into_inner();
 
         let last_executed_cid = last_proof.as_ref().map(|p| p.sequence_number()).unwrap_or(SeqNo::ZERO);
@@ -1659,7 +1653,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         };
 
         // finalize view change by broadcasting a PREPARE msg
-        consensus.finalize_view_change((header, message), &view, self, timeouts, log, node);
+        let consensus_result = consensus.finalize_view_change((header, message), &view, self, timeouts, log, node)?;
 
         // Update proto phase
         self.phase.replace(ProtoPhase::Init);
@@ -1669,10 +1663,10 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
             self.currently_adding.borrow_mut().clear();
 
-            SynchronizerStatus::NewViewJoinedQuorum(to_execute, node.unwrap())
+            SynchronizerStatus::NewViewJoinedQuorum(consensus_result, to_execute, node.unwrap())
         } else {
             // resume normal phase
-            SynchronizerStatus::NewView(to_execute)
+            SynchronizerStatus::NewView(consensus_result, to_execute)
         }
     }
 

From c52a3e4e7ab8be8ff2d678f81ac60fbb980546d7 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 9 Oct 2023 14:33:15 +0100
Subject: [PATCH 58/80] Synchronizer installed pre prepares also must be
 communicated to the decision log.

---
 .../src/bft/consensus/accessory/mod.rs        |  20 +-
 .../bft/consensus/accessory/replica/mod.rs    |   1 -
 .../src/bft/consensus/decision/mod.rs         |  93 ++-
 febft-pbft-consensus/src/bft/consensus/mod.rs | 161 +++--
 .../src/bft/log/decided/mod.rs                |   6 +-
 .../src/bft/log/deciding/mod.rs               |   7 +-
 .../src/bft/{msg_log => log}/decisions/mod.rs |  30 +-
 .../src/bft/{msg_log => log}/mod.rs           |  25 +-
 febft-pbft-consensus/src/bft/mod.rs           | 348 +++++------
 .../src/bft/msg_log/decided_log/mod.rs        | 275 --------
 .../src/bft/msg_log/deciding_log/mod.rs       | 588 ------------------
 febft-pbft-consensus/src/bft/sync/mod.rs      | 157 ++---
 12 files changed, 430 insertions(+), 1281 deletions(-)
 rename febft-pbft-consensus/src/bft/{msg_log => log}/decisions/mod.rs (93%)
 rename febft-pbft-consensus/src/bft/{msg_log => log}/mod.rs (54%)
 delete mode 100755 febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
 delete mode 100644 febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs

diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
index cc1aa6bc..bf4a0bf0 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/mod.rs
@@ -9,8 +9,6 @@ use crate::bft::consensus::accessory::replica::ReplicaAccessory;
 use crate::bft::log::deciding::WorkingDecisionLog;
 use crate::bft::message::ConsensusMessage;
 use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::msg_log::deciding_log::DecidingLog;
-use crate::bft::msg_log::decisions::StoredConsensusMessage;
 use crate::bft::sync::view::ViewInfo;
 
 pub mod replica;
@@ -21,7 +19,7 @@ pub enum ConsensusDecisionAccessory<D>
     Replica(ReplicaAccessory<D>),
 }
 
-pub trait AccessoryConsensus<D> where D: ApplicationData + 'static,{
+pub trait AccessoryConsensus<D> where D: ApplicationData + 'static, {
     /// Handle the reception of a pre-prepare message without having completed the pre prepare phase
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
@@ -63,7 +61,7 @@ pub trait AccessoryConsensus<D> where D: ApplicationData + 'static,{
 }
 
 impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
-    where D: ApplicationData + 'static{
+    where D: ApplicationData + 'static {
     fn handle_partial_pre_prepare<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                       view: &ViewInfo,
                                       header: &Header, msg: &ConsensusMessage<D::Request>,
@@ -71,7 +69,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_partial_pre_prepare(deciding_log, view, msg, node);
+                rep.handle_partial_pre_prepare(deciding_log, view, header, msg, node);
             }
         }
     }
@@ -79,11 +77,11 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
     fn handle_pre_prepare_phase_completed<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
                                               view: &ViewInfo,
                                               header: &Header, msg: &ConsensusMessage<D::Request>,
-                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static{
+                                              node: &Arc<NT>) where NT: OrderProtocolSendNode<D, PBFTConsensus<D>> + 'static {
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_pre_prepare_phase_completed(deciding_log, view, msg, node);
+                rep.handle_pre_prepare_phase_completed(deciding_log, view, header, msg, node);
             }
         }
     }
@@ -95,7 +93,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_preparing_no_quorum(deciding_log, view, msg, node);
+                rep.handle_preparing_no_quorum(deciding_log, view, header, msg, node);
             }
         }
     }
@@ -107,7 +105,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_preparing_quorum(deciding_log, view, msg, node);
+                rep.handle_preparing_quorum(deciding_log, view, header, msg, node);
             }
         }
     }
@@ -119,7 +117,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_committing_no_quorum(deciding_log, view, msg, node);
+                rep.handle_committing_no_quorum(deciding_log, view, header, msg, node);
             }
         }
     }
@@ -131,7 +129,7 @@ impl<D> AccessoryConsensus<D> for ConsensusDecisionAccessory<D>
         match self {
             ConsensusDecisionAccessory::Follower => {}
             ConsensusDecisionAccessory::Replica(rep) => {
-                rep.handle_committing_quorum(deciding_log, view, msg, node);
+                rep.handle_committing_quorum(deciding_log, view, header, msg, node);
             }
         }
     }
diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index a25f58d7..cba7cfb6 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -16,7 +16,6 @@ use crate::bft::{PBFT, SysMsg};
 use crate::bft::consensus::accessory::AccessoryConsensus;
 use crate::bft::log::deciding::WorkingDecisionLog;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
-use crate::bft::msg_log::decisions::StoredConsensusMessage;
 use crate::bft::sync::view::ViewInfo;
 
 pub struct ReplicaAccessory<D>
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 9b7cefe1..5a363e64 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -7,13 +7,15 @@ use chrono::Utc;
 use log::{debug, info, warn};
 
 use atlas_common::error::*;
-use atlas_common::globals::ReadOnly;
+use atlas_common::maybe_vec::MaybeVec;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_core::messages::ClientRqInfo;
+use atlas_core::ordering_protocol::{Decision, DecisionInfo};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog};
+use atlas_core::smr::smr_decision_log::ShareableMessage;
 use atlas_core::timeouts::Timeouts;
 use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_duration;
@@ -21,11 +23,10 @@ use atlas_metrics::metrics::metric_duration;
 use crate::bft::consensus::accessory::{AccessoryConsensus, ConsensusDecisionAccessory};
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
 use crate::bft::log::deciding::{CompletedBatch, WorkingDecisionLog};
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
+use crate::bft::log::decisions::{IncompleteProof, ProofMetadata};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{ConsensusMetrics, PRE_PREPARE_ANALYSIS_ID};
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::{IncompleteProof, StoredConsensusMessage};
 use crate::bft::PBFT;
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer};
 use crate::bft::sync::view::ViewInfo;
@@ -68,7 +69,7 @@ pub enum DecisionPollStatus<O> {
     // Receive a message from the network
     Recv,
     // We currently have a message to be processed at this time
-    NextMessage(Header, ConsensusMessage<O>),
+    NextMessage(ShareableMessage<PBFTMessage<O>>),
     // This consensus decision is finished and therefore can be finalized
     Decided,
 }
@@ -84,29 +85,29 @@ pub enum DecisionStatus<O> {
     MessageQueued,
     /// A `febft` quorum still hasn't made a decision
     /// on a client request to be executed.
-    Deciding(StoredMessage<ConsensusMessage<O>>),
+    Deciding(ShareableMessage<PBFTMessage<O>>),
     /// Transitioned to another next phase of the consensus decision
-    Transitioned(StoredMessage<ConsensusMessage<O>>),
+    Transitioned(ShareableMessage<PBFTMessage<O>>),
     /// A `febft` quorum decided on the execution of
     /// the batch of requests with the given digests.
     /// The first digest is the digest of the Prepare message
     /// And therefore the entire batch digest
     /// THe second Vec<Digest> is a vec with digests of the requests contained in the batch
     /// The third is the messages that should be persisted for this batch to be considered persisted
-    Decided(StoredMessage<ConsensusMessage<O>>),
+    Decided(ShareableMessage<PBFTMessage<O>>),
     DecidedIgnored,
 }
 
 /// A message queue for this particular consensus instance
 pub struct MessageQueue<O> {
     get_queue: bool,
-    pre_prepares: VecDeque<StoredMessage<ConsensusMessage<O>>>,
-    prepares: VecDeque<StoredMessage<ConsensusMessage<O>>>,
-    commits: VecDeque<StoredMessage<ConsensusMessage<O>>>,
+    pre_prepares: VecDeque<ShareableMessage<PBFTMessage<O>>>,
+    prepares: VecDeque<ShareableMessage<PBFTMessage<O>>>,
+    commits: VecDeque<ShareableMessage<PBFTMessage<O>>>,
 }
 
 /// The information needed to make a decision on a batch of requests.
-pub struct ConsensusDecision<D, PL> where D: ApplicationData + 'static, {
+pub struct ConsensusDecision<D> where D: ApplicationData + 'static, {
     node_id: NodeId,
     /// The sequence number of this consensus decision
     seq: SeqNo,
@@ -135,9 +136,9 @@ impl<O> MessageQueue<O> {
         }
     }
 
-    pub(super) fn from_messages(pre_prepares: VecDeque<StoredMessage<ConsensusMessage<O>>>,
-                                prepares: VecDeque<StoredMessage<ConsensusMessage<O>>>,
-                                commits: VecDeque<StoredMessage<ConsensusMessage<O>>>) -> Self {
+    pub(super) fn from_messages(pre_prepares: VecDeque<ShareableMessage<PBFTMessage<O>>>,
+                                prepares: VecDeque<ShareableMessage<PBFTMessage<O>>>,
+                                commits: VecDeque<ShareableMessage<PBFTMessage<O>>>) -> Self {
         let get_queue = !pre_prepares.is_empty() || !prepares.is_empty() || !commits.is_empty();
 
         Self {
@@ -154,28 +155,28 @@ impl<O> MessageQueue<O> {
 
     pub fn is_signalled(&self) -> bool { self.get_queue }
 
-    fn queue_pre_prepare(&mut self, message: StoredMessage<ConsensusMessage<O>>) {
+    fn queue_pre_prepare(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
         self.pre_prepares.push_back(message);
 
         self.signal();
     }
 
-    fn queue_prepare(&mut self, message: StoredMessage<ConsensusMessage<O>>) {
+    fn queue_prepare(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
         self.prepares.push_back(message);
 
         self.signal();
     }
 
-    fn queue_commit(&mut self, message: StoredMessage<ConsensusMessage<O>>) {
+    fn queue_commit(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
         self.commits.push_back(message);
 
         self.signal();
     }
 }
 
-impl<D, PL> ConsensusDecision<D, PL>
+impl<D> ConsensusDecision<D>
     where D: ApplicationData + 'static, {
-    pub fn init_decision(node_id: NodeId, seq_no: SeqNo, view: &ViewInfo, persistent_log: PL) -> Self {
+    pub fn init_decision(node_id: NodeId, seq_no: SeqNo, view: &ViewInfo) -> Self {
         Self {
             node_id,
             seq: seq_no,
@@ -188,7 +189,6 @@ impl<D, PL> ConsensusDecision<D, PL>
     }
 
     pub fn init_with_msg_log(node_id: NodeId, seq_no: SeqNo, view: &ViewInfo,
-                             persistent_log: PL,
                              message_queue: MessageQueue<D::Request>) -> Self {
         Self {
             node_id,
@@ -201,16 +201,16 @@ impl<D, PL> ConsensusDecision<D, PL>
         }
     }
 
-    pub fn queue(&mut self, header: Header, message: ConsensusMessage<D::Request>) {
+    pub fn queue(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) {
         match message.kind() {
             ConsensusMessageKind::PrePrepare(_) => {
-                self.message_queue.queue_pre_prepare(StoredMessage::new(header, message));
+                self.message_queue.queue_pre_prepare(message);
             }
             ConsensusMessageKind::Prepare(_) => {
-                self.message_queue.queue_prepare(StoredMessage::new(header, message));
+                self.message_queue.queue_prepare(message);
             }
             ConsensusMessageKind::Commit(_) => {
-                self.message_queue.queue_commit(StoredMessage::new(header, message));
+                self.message_queue.queue_commit(message);
             }
         }
     }
@@ -258,15 +258,14 @@ impl<D, PL> ConsensusDecision<D, PL>
 
     /// Process a message relating to this consensus instance
     pub fn process_message<NT>(&mut self,
-                               header: Header,
-                               message: ConsensusMessage<D::Request>,
+                               s_message: ShareableMessage<PBFTMessage<D::Request>>,
                                synchronizer: &Synchronizer<D>,
                                timeouts: &Timeouts,
-                               log: &mut Log<D, PL>,
                                node: &Arc<NT>) -> Result<DecisionStatus<D::Request>>
-        where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-              PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+        where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
         let view = synchronizer.view();
+        let header = s_message.header();
+        let message = s_message.message().consensus();
 
         return match self.phase {
             DecisionPhase::Initialize => {
@@ -274,7 +273,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                 // This consensus instance.
                 warn!("{:?} // Queueing message {:?} as we are in the initialize phase", self.node_id, message);
 
-                self.queue(header, message);
+                self.queue(s_message);
 
                 return Ok(DecisionStatus::MessageQueued);
             }
@@ -308,7 +307,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         debug!("{:?} // Received {:?} from {:?} while in prepreparing ",
                             self.node_id, message, header.from());
 
-                        self.message_queue.queue_prepare(StoredMessage::new(header, message));
+                        self.message_queue.queue_prepare(s_message);
 
                         return Ok(DecisionStatus::MessageQueued);
                     }
@@ -316,7 +315,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         debug!("{:?} // Received {:?} from {:?} while in pre preparing",
                             self.node_id, message, header.from());
 
-                        self.message_queue.queue_commit(StoredMessage::new(header, message));
+                        self.message_queue.queue_commit(s_message);
 
                         return Ok(DecisionStatus::MessageQueued);
                     }
@@ -334,13 +333,13 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                 //TODO: Try out cloning each request on this method,
                 let mut digests = request_batch_received(
-                    &message,
+                    message,
                     timeouts,
                     synchronizer,
                     &mut self.working_log,
                 );
 
-                let batch_metadata = self.working_log.process_pre_prepare(header.clone(), &message,
+                let batch_metadata = self.working_log.process_pre_prepare(header.clone(), message,
                                                                           header.digest().clone(), digests)?;
 
                 let mut result;
@@ -365,17 +364,13 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                     let current_digest = batch_metadata.batch_digest();
 
-                    // Register that all of the batches have been received
-                    // The digest of the batch and the order of the batches
-                    log.all_batches_received(batch_metadata);
-
                     self.accessory.handle_pre_prepare_phase_completed(&self.working_log,
                                                                       &view, &header, &message, node);
 
                     self.message_queue.signal();
 
                     // Mark that we have transitioned to the next phase
-                    result = DecisionStatus::Transitioned(StoredMessage::new(header, message));
+                    result = DecisionStatus::Transitioned(s_message);
 
                     // We no longer start the count at 1 since all leaders must also send the prepare
                     // message with the digest of the entire batch
@@ -387,7 +382,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.accessory.handle_partial_pre_prepare(&self.working_log, &view,
                                                               &header, &message, &**node);
 
-                    result = DecisionStatus::Deciding(StoredMessage::new(header, message));
+                    result = DecisionStatus::Deciding(s_message);
 
                     DecisionPhase::PrePreparing(received)
                 };
@@ -407,7 +402,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                         debug!("{:?} // Received {:?} from {:?} while in preparing phase",
                             self.node_id, message, header.from());
 
-                        self.message_queue.queue_commit(StoredMessage::new(header, message));
+                        self.message_queue.queue_commit(s_message);
 
                         return Ok(DecisionStatus::MessageQueued);
                     }
@@ -460,7 +455,7 @@ impl<D, PL> ConsensusDecision<D, PL>
 
                     self.message_queue.signal();
 
-                    result = DecisionStatus::Transitioned(StoredMessage::new(header, message));
+                    result = DecisionStatus::Transitioned(s_message);
 
                     DecisionPhase::Committing(0)
                 } else {
@@ -470,7 +465,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.accessory.handle_preparing_no_quorum(&self.working_log, &view,
                                                               &header, &message, &**node);
 
-                    result = DecisionStatus::Deciding(StoredMessage::new(header, message));
+                    result = DecisionStatus::Deciding(s_message);
 
                     DecisionPhase::Preparing(received)
                 };
@@ -529,7 +524,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.accessory.handle_committing_quorum(&self.working_log, &view,
                                                             &header, &message, &**node);
 
-                    Ok(DecisionStatus::Decided(StoredMessage::new(header, message)))
+                    Ok(DecisionStatus::Decided(s_message))
                 } else {
                     debug!("{:?} // Received commit message {:?} from {:?}. Current count {}",
                         self.node_id, stored_msg.message().sequence_number(), header.from(), received);
@@ -539,7 +534,7 @@ impl<D, PL> ConsensusDecision<D, PL>
                     self.accessory.handle_committing_no_quorum(&self.working_log, &view,
                                                                &header, &message, &**node);
 
-                    Ok(DecisionStatus::Deciding(StoredMessage::new(header, message)))
+                    Ok(DecisionStatus::Deciding(s_message))
                 };
             }
             DecisionPhase::Decided => {
@@ -577,7 +572,7 @@ impl<D, PL> ConsensusDecision<D, PL>
     }
 }
 
-impl<D, PL> Orderable for ConsensusDecision<D, PL>
+impl<D> Orderable for ConsensusDecision<D>
     where D: ApplicationData + 'static, {
     fn sequence_number(&self) -> SeqNo {
         self.seq
@@ -624,8 +619,8 @@ impl<O> Debug for DecisionPollStatus<O> {
             DecisionPollStatus::Recv => {
                 write!(f, "Recv")
             }
-            DecisionPollStatus::NextMessage(_, msg) => {
-                write!(f, "Next Message {:?}", msg)
+            DecisionPollStatus::NextMessage(message) => {
+                write!(f, "Next Message {:?}, Message Type {:?}", message.header(), message.message())
             }
             DecisionPollStatus::Decided => {
                 write!(f, "Decided")
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 29e3e2ba..326991d3 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -8,14 +8,17 @@ use event_listener::Event;
 use log::{debug, error, info, trace, warn};
 
 use atlas_common::error::*;
+use atlas_common::globals::ReadOnly;
+use atlas_common::maybe_vec::MaybeVec;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_advance_message_queue_return, tbo_queue_message};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_core::ordering_protocol::ProtocolConsensusDecision;
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
+use atlas_core::ordering_protocol::{Decision, ProtocolConsensusDecision};
+use atlas_core::persistent_log::{OrderingProtocolLog};
+use atlas_core::smr::smr_decision_log::ShareableMessage;
 use atlas_core::timeouts::Timeouts;
 use atlas_smr_application::ExecutorHandle;
 use atlas_smr_application::serialize::ApplicationData;
@@ -24,11 +27,10 @@ use atlas_metrics::metrics::metric_increment;
 use crate::bft::{PBFT, SysMsg};
 use crate::bft::consensus::decision::{ConsensusDecision, DecisionPollStatus, DecisionStatus, MessageQueue};
 use crate::bft::log::deciding::CompletedBatch;
+use crate::bft::log::decisions::{DecisionLog, IncompleteProof, Proof, ProofMetadata};
+use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
-use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::OPERATIONS_PROCESSED_ID;
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::{DecisionLog, IncompleteProof, Proof};
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer};
 use crate::bft::sync::view::ViewInfo;
 
@@ -37,7 +39,7 @@ pub mod accessory;
 
 #[derive(Debug, Clone)]
 /// Status returned from processing a consensus message.
-pub enum ConsensusStatus {
+pub enum ConsensusStatus<O> {
     /// A particular node tried voting twice.
     VotedTwice(NodeId),
     /// The message has been ignored
@@ -46,14 +48,14 @@ pub enum ConsensusStatus {
     MessageQueued,
     /// A `febft` quorum still hasn't made a decision
     /// on a client request to be executed.
-    Deciding,
+    Deciding(MaybeVec<Decision<ProofMetadata, PBFTMessage<O>, O>>),
     /// A `febft` quorum decided on the execution of
     /// the batch of requests with the given digests.
     /// The first digest is the digest of the Prepare message
     /// And therefore the entire batch digest
     /// THe second Vec<Digest> is a vec with digests of the requests contained in the batch
     /// The third is the messages that should be persisted for this batch to be considered persisted
-    Decided,
+    Decided(MaybeVec<Decision<ProofMetadata, PBFTMessage<O>, O>>),
 }
 
 #[derive(Debug, Clone)]
@@ -63,7 +65,7 @@ pub enum ConsensusPollStatus<O> {
     /// poll its main channel for more messages.
     Recv,
     /// A new consensus message is available to be processed.
-    NextMessage(Header, ConsensusMessage<O>),
+    NextMessage(ShareableMessage<PBFTMessage<O>>),
     /// The first consensus instance of the consensus queue is ready to be finalized
     /// as it has already been decided
     Decided,
@@ -79,9 +81,9 @@ pub struct TboQueue<O> {
     curr_seq: SeqNo,
     watermark: u32,
     get_queue: bool,
-    pre_prepares: VecDeque<VecDeque<StoredMessage<ConsensusMessage<O>>>>,
-    prepares: VecDeque<VecDeque<StoredMessage<ConsensusMessage<O>>>>,
-    commits: VecDeque<VecDeque<StoredMessage<ConsensusMessage<O>>>>,
+    pre_prepares: VecDeque<VecDeque<ShareableMessage<PBFTMessage<O>>>>,
+    prepares: VecDeque<VecDeque<ShareableMessage<PBFTMessage<O>>>>,
+    commits: VecDeque<VecDeque<ShareableMessage<PBFTMessage<O>>>>,
 }
 
 impl<O> Orderable for TboQueue<O> {
@@ -137,34 +139,34 @@ impl<O> TboQueue<O> {
 
     /// Queues a consensus message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
-    pub fn queue(&mut self, h: Header, m: ConsensusMessage<O>) {
-        match m.kind() {
-            ConsensusMessageKind::PrePrepare(_) => self.queue_pre_prepare(h, m),
-            ConsensusMessageKind::Prepare(_) => self.queue_prepare(h, m),
-            ConsensusMessageKind::Commit(_) => self.queue_commit(h, m),
+    pub fn queue(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
+        match message.message().consensus().kind() {
+            ConsensusMessageKind::PrePrepare(_) => self.queue_pre_prepare(message),
+            ConsensusMessageKind::Prepare(_) => self.queue_prepare(message),
+            ConsensusMessageKind::Commit(_) => self.queue_commit(message),
         }
     }
 
     /// Queues a `PRE-PREPARE` message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
-    fn queue_pre_prepare(&mut self, h: Header, m: ConsensusMessage<O>) {
+    fn queue_pre_prepare(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
         tbo_queue_message(
             self.base_seq(),
             &mut self.pre_prepares,
-            StoredMessage::new(h, m),
+            message,
         )
     }
 
     /// Queues a `PREPARE` message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
-    fn queue_prepare(&mut self, h: Header, m: ConsensusMessage<O>) {
-        tbo_queue_message(self.base_seq(), &mut self.prepares, StoredMessage::new(h, m))
+    fn queue_prepare(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
+        tbo_queue_message(self.base_seq(), &mut self.prepares, message)
     }
 
     /// Queues a `COMMIT` message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
-    fn queue_commit(&mut self, h: Header, m: ConsensusMessage<O>) {
-        tbo_queue_message(self.base_seq(), &mut self.commits, StoredMessage::new(h, m))
+    fn queue_commit(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
+        tbo_queue_message(self.base_seq(), &mut self.commits, message)
     }
 
     /// Clear this queue
@@ -188,9 +190,8 @@ pub struct Signals {
 
 /// The consensus handler. Responsible for multiplexing consensus instances and keeping track
 /// of missing messages
-pub struct Consensus<D, PL>
-    where D: ApplicationData + 'static,
-          PL: Clone {
+pub struct Consensus<D,>
+    where D: ApplicationData + 'static,{
     node_id: NodeId,
     /// The handle to the executor of the function
     executor_handle: ExecutorHandle<D>,
@@ -205,14 +206,14 @@ pub struct Consensus<D, PL>
     /// The consensus instances that are currently being processed
     /// A given consensus instance n will only be finished when all consensus instances
     /// j, where j < n have already been processed, in order to maintain total ordering
-    decisions: VecDeque<ConsensusDecision<D, PL>>,
+    decisions: VecDeque<ConsensusDecision<D>>,
     /// The queue for messages that sit outside the range seq_no + watermark
     /// These messages cannot currently be processed since they sit outside the allowed
     /// zone but they will be processed once the seq no moves forward enough to include them
     tbo_queue: TboQueue<D::Request>,
     /// This queue serves for us to keep track of messages we receive of coming up views.
     /// This is important for us to be able to continue the process of moving views after a view change
-    view_queue: VecDeque<Vec<StoredMessage<ConsensusMessage<D::Request>>>>,
+    view_queue: VecDeque<Vec<ShareableMessage<PBFTMessage<D::Request>>>>,
     /// The consensus guard that will be used to ensure that the proposer only proposes one batch
     /// for each consensus instance
     consensus_guard: Arc<ProposerConsensusGuard>,
@@ -220,15 +221,11 @@ pub struct Consensus<D, PL>
     timeouts: Timeouts,
     /// Check if we are currently recovering from a fault, meaning we should ignore timeouts
     is_recovering: bool,
-
-    persistent_log: PL,
 }
 
-impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
-                                   PL: Clone {
+impl<D> Consensus<D> where D: ApplicationData + 'static {
     pub fn new_replica(node_id: NodeId, view: &ViewInfo, executor_handle: ExecutorHandle<D>, seq_no: SeqNo,
-                       watermark: u32, consensus_guard: Arc<ProposerConsensusGuard>, timeouts: Timeouts,
-                       persistent_log: PL) -> Self {
+                       watermark: u32, consensus_guard: Arc<ProposerConsensusGuard>, timeouts: Timeouts) -> Self {
         let mut curr_seq = seq_no;
 
         let mut consensus = Self {
@@ -244,7 +241,6 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             consensus_guard,
             timeouts,
             is_recovering: false,
-            persistent_log,
         };
 
         // Initialize the consensus instances
@@ -253,7 +249,6 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
                 node_id,
                 curr_seq,
                 view,
-                consensus.persistent_log.clone(),
             );
 
             consensus.enqueue_decision(decision);
@@ -265,14 +260,16 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
     }
 
     /// Queue a given message into our message queues.
-    pub fn queue(&mut self, header: Header, message: ConsensusMessage<D::Request>) {
-        let message_seq = message.sequence_number();
+    pub fn queue(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) {
+        let message_seq = message.message().sequence_number();
 
-        let view_seq = message.view();
+        let view_seq = message.message().consensus().view();
+
+        let header = message.header();
 
         match view_seq.index(self.curr_view.sequence_number()) {
             Either::Right(i) if i > 0 => {
-                self.enqueue_other_view_message(i, header, message);
+                self.enqueue_other_view_message(i, message);
 
                 return;
             }
@@ -304,12 +301,13 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
 
             // We are not currently processing this consensus instance
             // so we need to queue the message
-            self.tbo_queue.queue(header, message);
+            self.tbo_queue.queue(message);
         } else {
             debug!("{:?} // Queueing message out of context msg {:?} received from {:?} into the corresponding decision {}",
                 self.node_id, message, header.from(), i);
+
             // Queue the message in the corresponding pending decision
-            self.decisions.get_mut(i).unwrap().queue(header, message);
+            self.decisions.get_mut(i).unwrap().queue(message);
 
             // Signal that we are ready to receive messages
             self.signalled.push_signalled(message_seq);
@@ -327,12 +325,12 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
                 let poll_result = self.decisions[index].poll();
 
                 match poll_result {
-                    DecisionPollStatus::NextMessage(header, message) => {
+                    DecisionPollStatus::NextMessage(message) => {
                         // We had a message pending, so it's possible that there are more messages
                         // Pending
                         self.signalled.push_signalled(seq_no);
 
-                        return ConsensusPollStatus::NextMessage(header, message);
+                        return ConsensusPollStatus::NextMessage(message);
                     }
                     DecisionPollStatus::TryPropose => {
                         // This won't generate a loop since only the first poll will actually return
@@ -364,21 +362,20 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
     }
 
     pub fn process_message<NT>(&mut self,
-                               header: Header,
-                               message: ConsensusMessage<D::Request>,
+                               s_message: ShareableMessage<PBFTMessage<D::Request>>,
                                synchronizer: &Synchronizer<D>,
                                timeouts: &Timeouts,
-                               log: &mut Log<D, PL>,
-                               node: &Arc<NT>) -> Result<ConsensusStatus>
-        where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-              PL: OrderingProtocolLog<D, PBFTConsensus<D>>, {
+                               node: &Arc<NT>) -> Result<ConsensusStatus<D::Request>>
+        where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
+        let (header, message) = (s_message.header(), s_message.message().consensus());
+
         let message_seq = message.sequence_number();
 
         let view_seq = message.view();
 
         match view_seq.index(self.curr_view.sequence_number()) {
             Either::Right(i) if i > 0 => {
-                self.enqueue_other_view_message(i, header, message);
+                self.enqueue_other_view_message(i, s_message);
 
                 return Ok(ConsensusStatus::MessageQueued);
             }
@@ -406,7 +403,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             // so we need to queue the message
             debug!("{:?} // Queueing message {:?} for seq no {:?}", self.node_id, message, message_seq);
 
-            self.tbo_queue.queue(header, message);
+            self.tbo_queue.queue(s_message);
 
             return Ok(ConsensusStatus::MessageQueued);
         }
@@ -414,28 +411,30 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
         // Get the correct consensus instance for this message
         let decision = self.decisions.get_mut(i).unwrap();
 
-        let status = decision.process_message(header, message, synchronizer, timeouts, log, node)?;
+        let decision_seq = decision.sequence_number();
+
+        let status = decision.process_message(s_message, synchronizer, timeouts,  node)?;
 
         Ok(match status {
             DecisionStatus::VotedTwice(node) => {
                 ConsensusStatus::VotedTwice(node)
             }
             DecisionStatus::Deciding(message) => {
-                ConsensusStatus::Deciding
+                ConsensusStatus::Deciding(MaybeVec::from_one(Decision::decision_info_from_message(decision_seq, message)))
             }
             DecisionStatus::MessageQueued => {
                 //When we transition phases, we may discover new messages
                 // That were in the queue, so we must be signalled again
-                self.signalled.push_signalled(message_seq);
+                self.signalled.push_signalled(decision_seq);
 
                 ConsensusStatus::MessageQueued
             }
             DecisionStatus::Transitioned(message) => {
                 //When we transition phases, we may discover new messages
                 // That were in the queue, so we must be signalled again
-                self.signalled.push_signalled(message_seq);
+                self.signalled.push_signalled(decision_seq);
 
-                ConsensusStatus::Deciding
+                ConsensusStatus::Deciding(MaybeVec::from_one(Decision::decision_info_from_message(decision_seq, message)))
             }
             DecisionStatus::Decided(message) => {
                 ConsensusStatus::Decided
@@ -494,7 +493,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
     /// Advance to the next instance of the consensus
     /// This will also create the necessary new decision to keep the pending decisions
     /// equal to the water mark
-    pub fn next_instance(&mut self, view: &ViewInfo) -> ConsensusDecision<D, PL> {
+    pub fn next_instance(&mut self, view: &ViewInfo) -> ConsensusDecision<D> {
         let decision = self.decisions.pop_front().unwrap();
 
         self.seq_no = self.seq_no.next();
@@ -518,8 +517,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
         // Create the decision to keep the queue populated
         let novel_decision = ConsensusDecision::init_with_msg_log(self.node_id,
                                                                   new_seq_no,
-                                                                  view,
-                                                                  self.persistent_log.clone(), queue, );
+                                                                  view, queue, );
 
         self.enqueue_decision(novel_decision);
 
@@ -592,7 +590,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
 
                 while self.decisions.len() < self.watermark as usize {
                     let novel_decision = ConsensusDecision::init_decision(self.node_id,
-                                                                          sequence_no, view, self.persistent_log.clone());
+                                                                          sequence_no, view);
 
                     self.enqueue_decision(novel_decision);
 
@@ -643,8 +641,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
                 while self.tbo_queue.sequence_number() < novel_seq_no && self.decisions.len() < self.watermark as usize {
                     let messages = self.tbo_queue.advance_queue();
 
-                    let decision = ConsensusDecision::init_with_msg_log(self.node_id, sequence_no, view,
-                                                                        self.persistent_log.clone(), messages);
+                    let decision = ConsensusDecision::init_with_msg_log(self.node_id, sequence_no, view,messages);
 
                     debug!("{:?} // Initialized new decision from TBO queue messages {:?}", self.node_id, decision.sequence_number());
 
@@ -654,8 +651,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
                 }
 
                 while self.decisions.len() < self.watermark as usize {
-                    let decision = ConsensusDecision::init_decision(self.node_id, sequence_no,
-                                                                    view, self.persistent_log.clone());
+                    let decision = ConsensusDecision::init_decision(self.node_id, sequence_no, view);
 
                     self.enqueue_decision(decision);
 
@@ -686,7 +682,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
                     let messages = self.tbo_queue.advance_queue();
 
                     let decision = ConsensusDecision::init_with_msg_log(self.node_id, sequence_no,
-                                                                        view, self.persistent_log.clone(), messages);
+                                                                        view, messages);
 
                     self.enqueue_decision(decision);
 
@@ -710,8 +706,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
                               seq: SeqNo,
                               view: &ViewInfo,
                               proof: Proof<D::Request>,
-                              log: &mut Log<D, PL>) -> Result<ProtocolConsensusDecision<D::Request>>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+                              log: &mut Log<D>) -> Result<ProtocolConsensusDecision<D::Request>> {
 
         // If this is successful, it means that we are all caught up and can now start executing the
         // batch
@@ -765,7 +760,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
         let mut sequence_no = self.sequence_number();
 
         while self.decisions.len() < self.watermark as usize {
-            let novel_decision = ConsensusDecision::init_decision(self.node_id, sequence_no, view, self.persistent_log.clone());
+            let novel_decision = ConsensusDecision::init_decision(self.node_id, sequence_no, view);
 
             self.enqueue_decision(novel_decision);
 
@@ -784,15 +779,13 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
 
         if let Some(messages) = option {
             for message in messages {
-                let (header, message) = message.into_inner();
-
-                self.queue(header, message);
+                self.queue(message);
             }
         }
     }
 
     /// Enqueue a message from another view into it's correct queue
-    fn enqueue_other_view_message(&mut self, index: usize, header: Header, message: ConsensusMessage<D::Request>) {
+    fn enqueue_other_view_message(&mut self, index: usize, message: ShareableMessage<PBFTMessage<D::Request>>) {
         debug!("{:?} // Enqueuing a message from another view into the view queue. Index {}  {:?}", self.node_id, index, message);
 
         // Adjust the index to be 0 based
@@ -802,7 +795,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
             self.view_queue.push_back(Vec::new());
         }
 
-        self.view_queue[index].push(StoredMessage::new(header, message));
+        self.view_queue[index].push(message);
     }
 
 
@@ -813,11 +806,10 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
         new_view: &ViewInfo,
         synchronizer: &Synchronizer<D>,
         timeouts: &Timeouts,
-        log: &mut Log<D, PL>,
+        log: &mut Log<D>,
         node: &Arc<NT>,
-    ) where
-        NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-        PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+    ) -> Result<ConsensusStatus<D::Request>> where
+        NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
         //Prepare the algorithm as we are already entering this phase
 
         self.install_view(new_view);
@@ -838,9 +830,13 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
         // So the proposer won't try to propose anything to this decision
         self.decisions[0].skip_init_phase();
 
-        self.process_message(header, message, synchronizer, timeouts, log, node).unwrap();
+        let shareable_message = Arc::new(ReadOnly::new(StoredMessage::new(header, PBFTMessage::Consensus(message))));
+
+        let result = self.process_message(shareable_message, synchronizer, timeouts, node)?;
 
         self.consensus_guard.unlock_consensus();
+
+        Ok(result)
     }
 
     /// Collect the incomplete proof that is currently being decided
@@ -853,7 +849,7 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
     }
 
     /// Enqueue a decision onto our overlapping decision log
-    fn enqueue_decision(&mut self, decision: ConsensusDecision<D, PL>) {
+    fn enqueue_decision(&mut self, decision: ConsensusDecision<D>) {
         self.signalled.push_signalled(decision.sequence_number());
 
         self.decisions.push_back(decision);
@@ -875,9 +871,8 @@ impl<D, PL> Consensus<D, PL> where D: ApplicationData + 'static,
     }
 }
 
-impl<D, PL> Orderable for Consensus<D, PL>
-    where D: ApplicationData + 'static,
-          PL: Clone {
+impl<D> Orderable for Consensus<D>
+    where D: ApplicationData + 'static, {
     fn sequence_number(&self) -> SeqNo {
         self.seq_no
     }
diff --git a/febft-pbft-consensus/src/bft/log/decided/mod.rs b/febft-pbft-consensus/src/bft/log/decided/mod.rs
index 5e9aeb75..019461a5 100644
--- a/febft-pbft-consensus/src/bft/log/decided/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decided/mod.rs
@@ -1,4 +1,4 @@
-use crate::bft::msg_log::decisions::Proof;
+use crate::bft::log::decisions::Proof;
 
 /// A necessary decision log for the ability to perform view changes.
 /// Only stores the latest performed decision
@@ -15,10 +15,12 @@ impl<O> OngoingDecisionLog<O> {
         }
     }
 
+    /// Install a given proof
     fn install_proof(&mut self, proof: Proof<O>) {
         self.last_decision = Some(proof)
     }
-    
+
+    /// Get the last decision
     fn last_decision(&self) -> Option<Proof<O>>  {
         self.last_decision.clone()
     }
diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
index 72eecf6a..46b31684 100644
--- a/febft-pbft-consensus/src/bft/log/deciding/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -10,10 +10,9 @@ use atlas_communication::message::Header;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
 use atlas_metrics::benchmarks::BatchMeta;
 use atlas_metrics::metrics::metric_duration;
+use crate::bft::log::decisions::{IncompleteProof, ProofMetadata};
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
 use crate::bft::metric::PRE_PREPARE_LOG_ANALYSIS_ID;
-use crate::bft::msg_log::deciding_log::{DecidingLog, FullBatch};
-use crate::bft::msg_log::decisions::{IncompleteProof, ProofMetadata};
 use crate::bft::sync::view::ViewInfo;
 
 /// Information about the completed batch, the contained requests and
@@ -103,7 +102,7 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
                                header: Header,
                                message: &ConsensusMessage<O>,
                                digest: Digest,
-                               mut batch_rq_digests: Vec<ClientRqInfo>) -> Result<Option<FullBatch>> {
+                               mut batch_rq_digests: Vec<ClientRqInfo>) -> Result<Option<ProofMetadata>> {
         let start = Instant::now();
 
         let sending_leader = header.from();
@@ -160,7 +159,7 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
 
             self.batch_digest = Some(digest.clone());
 
-            Some(ProofMetadata::new(self.seq_no, digest, ordering))
+            Some(ProofMetadata::new(self.seq_no, digest, ordering, self.current_batch_size))
         } else {
             None
         })
diff --git a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
similarity index 93%
rename from febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
rename to febft-pbft-consensus/src/bft/log/decisions/mod.rs
index 4681306c..4877750b 100644
--- a/febft-pbft-consensus/src/bft/msg_log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
@@ -11,11 +11,11 @@ use atlas_common::globals::ReadOnly;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::StoredMessage;
 use atlas_core::ordering_protocol::networking::serialize::{OrderProtocolLog, OrderProtocolProof};
+use atlas_core::smr::smr_decision_log::ShareableMessage;
+use crate::bft::log::deciding::CompletedBatch;
 
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
-use crate::bft::msg_log::deciding_log::CompletedBatch;
-
-pub type StoredConsensusMessage<O> = Arc<ReadOnly<StoredMessage<ConsensusMessage<O>>>>;
+pub type StoredConsensusMessage<O> = ShareableMessage<PBFTMessage<O>>;
 
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone)]
@@ -47,6 +47,7 @@ pub struct ProofMetadata {
     seq_no: SeqNo,
     batch_digest: Digest,
     pre_prepare_ordering: Vec<Digest>,
+    contained_client_rqs: usize
 }
 
 impl Orderable for ProofMetadata {
@@ -100,11 +101,12 @@ impl PrepareSet {
 
 impl ProofMetadata {
     /// Create a new proof metadata
-    pub(crate) fn new(seq_no: SeqNo, digest: Digest, pre_prepare_ordering: Vec<Digest>) -> Self {
+    pub(crate) fn new(seq_no: SeqNo, digest: Digest, pre_prepare_ordering: Vec<Digest>, contained_rqs: usize) -> Self {
         Self {
             seq_no,
             batch_digest: digest,
             pre_prepare_ordering,
+            contained_client_rqs: contained_rqs
         }
     }
 
@@ -119,6 +121,10 @@ impl ProofMetadata {
     pub fn pre_prepare_ordering(&self) -> &Vec<Digest> {
         &self.pre_prepare_ordering
     }
+
+    pub fn contained_client_rqs(&self) -> usize {
+        self.contained_client_rqs
+    }
 }
 
 impl<O> Proof<O> {
@@ -163,6 +169,22 @@ impl<O> Proof<O> {
         Ok(())
     }
 
+    pub fn all_messages(&self) -> Vec<StoredConsensusMessage<O>> {
+        let mut messages = Vec::with_capacity(self.pre_prepares.len() + self.prepares.len() + self.commits.len());
+
+        for pre_prepare in self.pre_prepares() {
+            messages.push(pre_prepare.clone())
+        }
+        for prepare in self.prepares() {
+            messages.push(prepare.clone())
+        }
+        for commit in self.commits() {
+            messages.push(commit.clone())
+        }
+
+        messages
+    }
+
     /// Check if the pre prepares stored here are ordered correctly according
     /// to the [pre_prepare_ordering] in this same proof.
     pub fn are_pre_prepares_ordered(&self) -> Result<bool> {
diff --git a/febft-pbft-consensus/src/bft/msg_log/mod.rs b/febft-pbft-consensus/src/bft/log/mod.rs
similarity index 54%
rename from febft-pbft-consensus/src/bft/msg_log/mod.rs
rename to febft-pbft-consensus/src/bft/log/mod.rs
index c6dd978a..f1916a80 100644
--- a/febft-pbft-consensus/src/bft/msg_log/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/mod.rs
@@ -1,23 +1,24 @@
-//! A module to manage the `febft` message log.
-
-use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::SeqNo;
 use atlas_communication::message::Header;
 use atlas_core::messages::RequestMessage;
 use atlas_smr_application::serialize::ApplicationData;
+use crate::bft::log::decisions::DecisionLog;
 
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::DecisionLog;
-
+pub mod decided;
+pub mod deciding;
 pub mod decisions;
-pub mod deciding_log;
-pub mod decided_log;
 
-pub fn initialize_decided_log<D: ApplicationData + 'static, PL>(node_id: NodeId,
-                                                                persistent_log: PL,
-                                                                state: Option<DecisionLog<D::Request>>) -> Result<Log<D, PL>> {
-    Ok(Log::init_decided_log(node_id, persistent_log, state))
+pub struct Log<D> where D: ApplicationData {
+    decided: DecisionLog<D::Request>,
+}
+
+impl<D> Log<D> where D: ApplicationData {
+
+    pub fn decision_log(&self) -> &DecisionLog<D::Request> {
+        &self.decided
+    }
+
 }
 
 #[inline]
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index c9aa28de..0091bdc6 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -10,48 +10,44 @@ use std::sync::atomic::AtomicBool;
 use std::time::Instant;
 use ::log::{debug, info, trace, warn};
 
-use log::{debug, info, trace, warn};
-
 use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
+use atlas_common::maybe_vec::MaybeVec;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::serialize::Serializable;
-use atlas_core::messages::Protocol;
-use atlas_core::ordering_protocol::{ OrderingProtocol, OrderingProtocolArgs, OrderProtocolExecResult, OrderProtocolPoll, OrderProtocolTolerance, PermissionedOrderingProtocol, ProtocolConsensusDecision, SerProof, SerProofMetadata, View};
+use atlas_core::messages::{ClientRqInfo, Protocol};
+use atlas_core::ordering_protocol::{Decision, DecisionInfo, DecisionMetadata, JoinInfo, OPExecResult, OPPollResult, OrderingProtocol, OrderingProtocolArgs, OrderProtocolTolerance, PermissionedOrderingProtocol, ProtocolConsensusDecision, ProtocolMessage, View};
+use atlas_core::ordering_protocol::loggable::{LoggableOrderProtocol, PProof};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::networking::serialize::{NetworkView, OrderingProtocolMessage};
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::{ReconfigurableOrderProtocol, ReconfigurationAttemptResult};
-use atlas_core::persistent_log::{OrderingProtocolLog, PersistableOrderProtocol, StatefulOrderingProtocolLog};
 use atlas_core::reconfiguration_protocol::ReconfigurationProtocol;
 use atlas_core::request_pre_processing::RequestPreProcessor;
 use atlas_core::serialize::ReconfigurationProtocolMessage;
+use atlas_core::smr::smr_decision_log::{ShareableConsensusMessage, ShareableMessage};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_smr_application::ExecutorHandle;
 use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_duration;
+use atlas_smr_application::app::UpdateBatch;
 
 use crate::bft::config::PBFTConfig;
 use crate::bft::consensus::{Consensus, ConsensusPollStatus, ConsensusStatus, ProposerConsensusGuard};
+use crate::bft::log::decisions::{Proof, ProofMetadata};
+use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, ObserveEventKind, PBFTMessage, ViewChangeMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{CONSENSUS_INSTALL_STATE_TIME_ID, MSG_LOG_INSTALL_TIME_ID};
-use crate::bft::msg_log::initialize_decided_log;
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::{DecisionLog, Proof};
 use crate::bft::proposer::Proposer;
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer, SynchronizerPollStatus, SynchronizerStatus, SyncReconfigurationResult};
 
 pub mod consensus;
 pub mod proposer;
 pub mod sync;
-pub mod msg_log;
-pub mod log {
-    pub mod deciding;
-    pub mod decided;
-}
+pub mod log;
 pub mod config;
 pub mod message;
 pub mod observer;
@@ -81,16 +77,18 @@ pub enum SyncPhaseRes<O> {
     RunCSTProtocol,
 }
 
+pub type OPDecision<O> = Decision<ProofMetadata, PBFTMessage<O>, O>;
+pub type OPDecisionInfo<O> = DecisionInfo<ProofMetadata, PBFTMessage<O>, O>;
+
 /// a PBFT based ordering protocol
-pub struct PBFTOrderProtocol<D, NT, PL>
+pub struct PBFTOrderProtocol<D, NT, >
     where
         D: ApplicationData + 'static,
-        NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-        PL: Clone {
+        NT: OrderProtocolSendNode<D, PBFT<D>> + 'static, {
     // What phase of the consensus algorithm are we currently executing
     phase: ConsensusPhase,
     /// The consensus state machine
-    consensus: Consensus<D, PL>,
+    consensus: Consensus<D>,
     /// The synchronizer state machine
     synchronizer: Arc<Synchronizer<D>>,
     /// The request pre processor
@@ -105,7 +103,7 @@ pub struct PBFTOrderProtocol<D, NT, PL>
     // The log of the decided consensus messages
     // This is completely owned by the server thread and therefore does not
     // Require any synchronization
-    message_log: Log<D, PL>,
+    message_log: Log<D>,
     // The proposer of this replica
     proposer: Arc<Proposer<D, NT>>,
     // The networking layer for a Node in the network (either Client or Replica)
@@ -114,28 +112,25 @@ pub struct PBFTOrderProtocol<D, NT, PL>
     executor: ExecutorHandle<D>,
 }
 
-impl<D, NT, PL> Orderable for PBFTOrderProtocol<D, NT, PL>
+impl<D, NT, > Orderable for PBFTOrderProtocol<D, NT>
     where D: 'static + ApplicationData,
-          NT: 'static + OrderProtocolSendNode<D, PBFT<D>>,
-          PL: Clone {
+          NT: 'static + OrderProtocolSendNode<D, PBFT<D>>, {
     fn sequence_number(&self) -> SeqNo {
         self.consensus.sequence_number()
     }
 }
 
-impl<D, NT, PL> OrderProtocolTolerance for PBFTOrderProtocol<D, NT, PL>
+impl<D, NT, > OrderProtocolTolerance for PBFTOrderProtocol<D, NT>
     where D: 'static + ApplicationData,
-          NT: 'static + OrderProtocolSendNode<D, PBFT<D>>,
-          PL: Clone, {
+          NT: 'static + OrderProtocolSendNode<D, PBFT<D>>, {
     fn get_n_for_f(f: usize) -> usize {
         3 * f + 1
     }
 }
 
-impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
+impl<D, NT> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT>
     where D: ApplicationData + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-          PL: Clone {
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static, {
     type Serialization = PBFTConsensus<D>;
     type Config = PBFTConfig<D>;
 
@@ -146,18 +141,15 @@ impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
     }
 
 
-    fn handle_off_ctx_message(&mut self, message: StoredMessage<Protocol<PBFTMessage<D::Request>>>)
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        let (header, message) = message.into_inner();
-
-        match message.into_inner() {
+    fn handle_off_ctx_message(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) {
+        match message.message() {
             PBFTMessage::Consensus(consensus) => {
                 debug!("{:?} // Received off context consensus message {:?}", self.node.id(), consensus);
-                self.consensus.queue(header, consensus);
+                self.consensus.queue(message);
             }
             PBFTMessage::ViewChange(view_change) => {
                 debug!("{:?} // Received off context view change message {:?}", self.node.id(), view_change);
-                self.synchronizer.queue(header, view_change);
+                self.synchronizer.queue(message);
 
                 self.synchronizer.signal();
             }
@@ -182,8 +174,7 @@ impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
         Ok(())
     }
 
-    fn poll(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+    fn poll(&mut self) -> OPPollResult<ProofMetadata, PBFTMessage<D::Request>, D::Request> {
         trace!("{:?} // Polling {:?}", self.node.id(), self.phase);
 
         match self.phase {
@@ -196,8 +187,7 @@ impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
         }
     }
 
-    fn process_message(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+    fn process_message(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>> {
         match self.phase {
             ConsensusPhase::NormalPhase => {
                 self.update_normal_phase(message)
@@ -208,31 +198,17 @@ impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
         }
     }
 
-    fn sequence_number_with_proof(&self) -> Result<Option<(SeqNo, SerProof<D, Self::Serialization>)>> {
-        Ok(self.message_log.last_proof(self.synchronizer.view().f())
-            .map(|p| (p.sequence_number(), p)))
-    }
-
-    fn verify_sequence_number(&self, seq_no: SeqNo, proof: &SerProof<D, Self::Serialization>) -> Result<bool> {
-        let proof: &Proof<D::Request> = proof;
-
-        //TODO: Verify the proof
-
-        Ok(true)
-    }
-
     fn install_seq_no(&mut self, seq_no: SeqNo) -> Result<()> {
         self.consensus.install_sequence_number(seq_no, &self.synchronizer.view());
 
         Ok(())
     }
 
-    fn handle_timeout(&mut self, timeout: Vec<RqTimeout>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+    fn handle_timeout(&mut self, timeout: Vec<RqTimeout>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>> {
         if self.consensus.is_catching_up() {
             warn!("{:?} // Ignoring timeouts while catching up", self.node.id());
 
-            return Ok(OrderProtocolExecResult::Success);
+            return Ok(OPExecResult::MessageDropped);
         }
 
         let status = self.synchronizer.client_requests_timed_out(self.node.id(), &timeout);
@@ -263,15 +239,13 @@ impl<D, NT, PL> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT, PL>
             _ => (),
         }
 
-        Ok(OrderProtocolExecResult::Success)
+        Ok(OPExecResult::MessageProcessedNoUpdate)
     }
 }
 
-impl<D, NT, PL> PermissionedOrderingProtocol for PBFTOrderProtocol<D, NT, PL>
+impl<D, NT> PermissionedOrderingProtocol for PBFTOrderProtocol<D, NT>
     where D: ApplicationData + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-          PL: Clone {
-
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
     type PermissionedSerialization = PBFTConsensus<D>;
 
     fn view(&self) -> View<Self::PermissionedSerialization> {
@@ -285,10 +259,9 @@ impl<D, NT, PL> PermissionedOrderingProtocol for PBFTOrderProtocol<D, NT, PL>
     }
 }
 
-impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
+impl<D, NT> PBFTOrderProtocol<D, NT>
     where D: ApplicationData + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-          PL: Clone {
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
     fn initialize_protocol(config: PBFTConfig<D>, args: OrderingProtocolArgs<D, NT>,
                            initial_state: Option<DecisionLog<D::Request>>) -> Result<Self> {
         let PBFTConfig {
@@ -300,17 +273,17 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
 
         let OrderingProtocolArgs(executor, timeouts,
                                  pre_processor, batch_input,
-                                 node,  quorum) = args;
+                                 node, quorum) = args;
 
         let sync = Synchronizer::initialize_with_quorum(node_id, view.sequence_number(), quorum.clone(), timeout_dur)?;
 
         let consensus_guard = ProposerConsensusGuard::new(sync.view(), watermark);
 
-        let consensus = Consensus::<D, PL>::new_replica(node_id, &sync.view(), executor.clone(),
-                                                        SeqNo::ZERO, watermark, consensus_guard.clone(),
-                                                        timeouts.clone(), persistent_log.clone());
+        let consensus = Consensus::<D>::new_replica(node_id, &sync.view(), executor.clone(),
+                                                    SeqNo::ZERO, watermark, consensus_guard.clone(),
+                                                    timeouts.clone());
 
-        let dec_log = initialize_decided_log::<D, PL>(node_id, persistent_log, initial_state)?;
+        let dec_log = initialize_decided_log::<D, PL>(node_id, initial_state)?;
 
         let proposer = Proposer::<D, NT>::new(node.clone(), batch_input, sync.clone(), timeouts.clone(),
                                               executor.clone(), consensus_guard.clone(),
@@ -349,8 +322,7 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         Ok(replica)
     }
 
-    fn poll_sync_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+    fn poll_sync_phase(&mut self) -> OPPollResult<ProofMetadata, PBFTMessage<D::Request>, D::Request> {
 
         // retrieve a view change message to be processed
         let poll_result = self.synchronizer.poll();
@@ -358,9 +330,9 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         debug!("{:?} // Polling sync phase {:?}", self.node.id(), poll_result);
 
         match poll_result {
-            SynchronizerPollStatus::Recv => OrderProtocolPoll::ReceiveFromReplicas,
-            SynchronizerPollStatus::NextMessage(h, m) => {
-                OrderProtocolPoll::Exec(StoredMessage::new(h, PBFTMessage::ViewChange(m)))
+            SynchronizerPollStatus::Recv => OPPollResult::ReceiveMsg,
+            SynchronizerPollStatus::NextMessage(message) => {
+                OPPollResult::Exec(message)
             }
             SynchronizerPollStatus::ResumeViewChange => {
                 debug!("{:?} // Resuming view change", self.node.id());
@@ -376,21 +348,34 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
 
                 if let Some(sync_status) = sync_status {
                     match sync_status {
-                        SynchronizerStatus::NewViewJoinedQuorum(decisions, node) => {
+                        SynchronizerStatus::NewViewJoinedQuorum(consensus_status, decisions, node) => {
                             let quorum_members = self.synchronizer.view().quorum_members().clone();
-                            return OrderProtocolPoll::QuorumJoined(decisions.map(|dec| vec![dec]), node, quorum_members);
+
+                            let decisions;
+
+                            match consensus_status {
+                                ConsensusStatus::VotedTwice(_) | ConsensusStatus::MessageQueued | ConsensusStatus::MessageIgnored => {
+                                    decisions = None;
+                                }
+                                ConsensusStatus::Deciding(decision) | ConsensusStatus::Decided(decision) => {
+                                    decisions = Some(decision)
+                                }
+                            }
+
+                            let joined = JoinInfo::new(node, self.synchronizer.view().quorum_members().clone());
+
+                            return OPPollResult::QuorumJoined(decisions, joined);
                         }
                         _ => {}
                     }
                 }
 
-                OrderProtocolPoll::RePoll
+                OPPollResult::RePoll
             }
         }
     }
 
-    fn poll_normal_phase(&mut self) -> OrderProtocolPoll<PBFTMessage<D::Request>, D::Request>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+    fn poll_normal_phase(&mut self) -> OPPollResult<ProofMetadata, PBFTMessage<D::Request>, D::Request> {
         // check if we have STOP messages to be processed,
         // and update our phase when we start installing
         // the new view
@@ -398,34 +383,34 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         while self.synchronizer.can_process_stops() {
             let sync_protocol = self.poll_sync_phase();
 
-            if let OrderProtocolPoll::Exec(message) = sync_protocol {
-                let (header, message) = message.into_inner();
+            if let OPPollResult::Exec(s_message) = sync_protocol {
+                let (header, message) = (s_message.header(), s_message.message());
 
                 if let PBFTMessage::ViewChange(view_change) = message.into_inner() {
-                    let result = self.adv_sync(header, view_change);
+                    let result = self.adv_sync(s_message);
 
                     return match result {
                         SyncPhaseRes::RunSyncProtocol => {
                             self.switch_phase(ConsensusPhase::SyncPhase);
 
-                            OrderProtocolPoll::RePoll
+                            OPPollResult::RePoll
                         }
                         SyncPhaseRes::RunCSTProtocol => {
                             // We don't need to switch to the sync phase
                             // As that has already been done by the adv sync method
-                            OrderProtocolPoll::RunCst
+                            OPPollResult::RunCst
                         }
                         SyncPhaseRes::SyncProtocolNotNeeded => {
                             warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, SyncProtocolNotNeeded");
-                            OrderProtocolPoll::RePoll
+                            OPPollResult::RePoll
                         }
                         SyncPhaseRes::JoinedQuorum(_, _) => {
                             warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, JoinedQuorum");
-                            OrderProtocolPoll::RePoll
+                            OPPollResult::RePoll
                         }
                         SyncPhaseRes::SyncProtocolFinished(_) => {
                             warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, Protocol Finished");
-                            OrderProtocolPoll::RePoll
+                            OPPollResult::RePoll
                         }
                     };
                 } else {
@@ -443,36 +428,36 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         let polled_message = self.consensus.poll();
 
         match polled_message {
-            ConsensusPollStatus::Recv => OrderProtocolPoll::ReceiveFromReplicas,
-            ConsensusPollStatus::NextMessage(h, m) => {
-                OrderProtocolPoll::Exec(StoredMessage::new(h, PBFTMessage::Consensus(m)))
+            ConsensusPollStatus::Recv => OPPollResult::ReceiveMsg,
+            ConsensusPollStatus::NextMessage(message) => {
+                OPPollResult::Exec(message)
             }
-            ConsensusPollStatus::Decided => {
-                return OrderProtocolPoll::Decided(self.finalize_all_possible().expect("Failed to finalize decisions"));
+            ConsensusPollStatus::Decided(decisions) => {
+                let finalized_decisions = self.finalize_all_possible().expect("Failed to finalize decisions");
+
+                return OPPollResult::Decided(finalized_decisions);
             }
         }
     }
 
-    fn update_sync_phase(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        let (header, protocol) = message.into_inner();
+    fn update_sync_phase(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>>{
 
-        match protocol.into_inner() {
+        match message.message() {
             PBFTMessage::ViewChange(view_change) => {
-                return Ok(match self.adv_sync(header, view_change) {
+                return Ok(match self.adv_sync(message) {
                     SyncPhaseRes::SyncProtocolNotNeeded => {
-                        OrderProtocolExecResult::Success
+                        OPExecResult::MessageProcessedNoUpdate
                     }
                     SyncPhaseRes::RunSyncProtocol => {
-                        OrderProtocolExecResult::Success
+                        OPExecResult::MessageProcessedNoUpdate
                     }
                     SyncPhaseRes::SyncProtocolFinished(to_execute) => {
                         match to_execute {
                             None => {
-                                OrderProtocolExecResult::Success
+                                OPExecResult::MessageProcessedNoUpdate
                             }
                             Some(to_execute) => {
-                                OrderProtocolExecResult::Decided(vec![to_execute])
+                                OPExecResult::Decided(vec![to_execute])
                             }
                         }
                     }
@@ -481,34 +466,31 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
 
                         let new_quorum = self.synchronizer.view().quorum_members().clone();
 
-                        OrderProtocolExecResult::QuorumJoined(to_execute.map(|x| vec![x]), node, new_quorum)
+                        OPExecResult::QuorumJoined(to_execute.map(|x| vec![x]), node, new_quorum)
                     }
                     SyncPhaseRes::RunCSTProtocol => {
-                        OrderProtocolExecResult::RunCst
+                        OPExecResult::RunCst
                     }
                 });
             }
-            PBFTMessage::Consensus(message) => {
-                self.consensus.queue(header, message);
+            PBFTMessage::Consensus(_) => {
+                self.consensus.queue(message);
             }
             _ => {}
         }
 
-        Ok(OrderProtocolExecResult::Success)
+        Ok(OPExecResult::MessageProcessedNoUpdate)
     }
 
-    fn update_normal_phase(&mut self, message: StoredMessage<PBFTMessage<D::Request>>) -> Result<OrderProtocolExecResult<D::Request>>
+    fn update_normal_phase(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>>
         where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        let (header, protocol) = message.into_inner();
-
-        match protocol {
-            PBFTMessage::Consensus(message) => {
-                return self.adv_consensus(header, message);
+        match message.message() {
+            PBFTMessage::Consensus(_) => {
+                return self.adv_consensus(message);
             }
-            PBFTMessage::ViewChange(view_change) => {
+            PBFTMessage::ViewChange(_) => {
                 let status = self.synchronizer.process_message(
-                    header,
-                    view_change,
+                    message,
                     &self.timeouts,
                     &mut self.message_log,
                     &self.pre_processor,
@@ -532,16 +514,14 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
             _ => {}
         }
 
-        Ok(OrderProtocolExecResult::Success)
+        Ok(OPExecResult::MessageProcessedNoUpdate)
     }
 
     /// Advance the consensus phase with a received message
     fn adv_consensus(
         &mut self,
-        header: Header,
-        message: ConsensusMessage<D::Request>,
-    ) -> Result<OrderProtocolExecResult<D::Request>>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+        message: ShareableMessage<PBFTMessage<D::Request>>,
+    ) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>> {
         let seq = self.consensus.sequence_number();
 
         // debug!(
@@ -551,37 +531,34 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
         // );
 
         let status = self.consensus.process_message(
-            header,
             message,
             &self.synchronizer,
             &self.timeouts,
-            &mut self.message_log,
             &self.node,
         )?;
 
-        match status {
-            // if deciding, nothing to do
-            ConsensusStatus::Deciding => {}
-            // FIXME: implement this
-            ConsensusStatus::VotedTwice(_) => todo!(),
-            // reached agreement, execute requests
-            //
-            // FIXME: execution layer needs to receive the id
-            // attributed by the consensus layer to each op,
-            // to execute in order
-            ConsensusStatus::Decided => {
-                return Ok(OrderProtocolExecResult::Decided(self.finalize_all_possible()?));
+        return Ok(match status {
+            ConsensusStatus::VotedTwice(_) | ConsensusStatus::MessageIgnored => {
+                OPExecResult::MessageDropped
             }
-        }
+            ConsensusStatus::MessageQueued => {
+                OPExecResult::MessageQueued
+            }
+            ConsensusStatus::Deciding(result) => {
+                OPExecResult::ProgressedDecision(result)
+            }
+            ConsensusStatus::Decided(result) => {
+                let finalized_decisions = self.finalize_all_possible()?;
 
-        //
-        // debug!(
-        //     "{:?} // Done processing consensus message. Took {:?}",
-        //     self.id(),
-        //     Instant::now().duration_since(start)
-        // );
+                let mut decision_builder = MaybeVec::builder();
+
+                for decision in finalized_decisions.into_iter() {
+                    decision_builder.push(Decision::completed_decision(decision.sequence_number(), decision));
+                }
 
-        Ok(OrderProtocolExecResult::Success)
+                OPExecResult::Decided(decision_builder.build())
+            }
+        });
     }
 
     /// Finalize all possible consensus instances
@@ -607,11 +584,8 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
 
 
     /// Advance the sync phase of the algorithm
-    fn adv_sync(&mut self, header: Header,
-                message: ViewChangeMessage<D::Request>) -> SyncPhaseRes<D::Request>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+    fn adv_sync(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) -> SyncPhaseRes<D::Request> {
         let status = self.synchronizer.process_message(
-            header,
             message,
             &self.timeouts,
             &mut self.message_log,
@@ -662,14 +636,13 @@ impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
     }
 }
 
-impl<D, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL>
+impl<D, NT, > StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT>
     where D: ApplicationData + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-          PL: Clone {
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
     type StateSerialization = PBFTConsensus<D>;
 
     fn initialize_with_initial_state(config: Self::Config,
-                                     args: OrderingProtocolArgs<D, NT, PL>,
+                                     args: OrderingProtocolArgs<D, NT>,
                                      initial_state: DecisionLog<D::Request>) -> Result<Self> where Self: Sized {
         Self::initialize_protocol(config, args, Some(initial_state))
     }
@@ -727,10 +700,9 @@ impl<D, NT, PL> StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT, PL
     }
 }
 
-impl<D, NT, PL> PBFTOrderProtocol<D, NT, PL>
+impl<D, NT> PBFTOrderProtocol<D, NT>
     where D: ApplicationData + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-          PL: Clone {
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static, {
     pub(crate) fn switch_phase(&mut self, new_phase: ConsensusPhase) {
         info!("{:?} // Switching from phase {:?} to phase {:?}", self.node.id(), self.phase, new_phase);
 
@@ -789,9 +761,11 @@ const CF_PRE_PREPARES: &str = "PRE_PREPARES";
 const CF_PREPARES: &str = "PREPARES";
 const CF_COMMIT: &str = "COMMITS";
 
-impl<D, NT, PL> PersistableOrderProtocol<D, PBFTConsensus<D>, PBFTConsensus<D>> for PBFTOrderProtocol<D, NT, PL>
+impl<D, NT> LoggableOrderProtocol<D, NT> for PBFTOrderProtocol<D, NT>
     where D: ApplicationData + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D>>, PL: Clone {
+          NT: OrderProtocolSendNode<D, PBFT<D>> {
+    type PersistableTypes = ();
+
     fn message_types() -> Vec<&'static str> {
         vec![
             CF_PRE_PREPARES,
@@ -800,21 +774,29 @@ impl<D, NT, PL> PersistableOrderProtocol<D, PBFTConsensus<D>, PBFTConsensus<D>>
         ]
     }
 
-    fn get_type_for_message(msg: &LoggableMessage<D, PBFTConsensus<D>>) -> Result<&'static str> {
-        match msg.kind() {
-            ConsensusMessageKind::PrePrepare(_) => {
-                Ok(CF_PRE_PREPARES)
-            }
-            ConsensusMessageKind::Prepare(_) => {
-                Ok(CF_PREPARES)
+    fn get_type_for_message(msg: &PBFTMessage<D::Request>) -> Result<&'static str> {
+        match msg {
+            PBFTMessage::Consensus(consensus) => {
+                match consensus.kind() {
+                    ConsensusMessageKind::PrePrepare(_) => {
+                        Ok(CF_PRE_PREPARES)
+                    }
+                    ConsensusMessageKind::Prepare(_) => {
+                        Ok(CF_PREPARES)
+                    }
+                    ConsensusMessageKind::Commit(_) => {
+                        Ok(CF_COMMIT)
+                    }
+                }
             }
-            ConsensusMessageKind::Commit(_) => {
-                Ok(CF_COMMIT)
+            PBFTMessage::ViewChange(view_change) => {
+                Err(Error::simple_with_msg(ErrorKind::Consensus, "Failed to get type for view change message."))
             }
+            PBFTMessage::ObserverMessage(_) => {}
         }
     }
 
-    fn init_proof_from(metadata: SerProofMetadata<D, PBFTConsensus<D>>, messages: Vec<StoredMessage<LoggableMessage<D, PBFTConsensus<D>>>>) -> SerProof<D, PBFTConsensus<D>> {
+    fn init_proof_from(metadata: ProofMetadata, messages: Vec<StoredMessage<PBFTMessage<D::Request>>>) -> Proof<D::Request> {
         let mut pre_prepares = Vec::with_capacity(messages.len() / 2);
         let mut prepares = Vec::with_capacity(messages.len() / 2);
         let mut commits = Vec::with_capacity(messages.len() / 2);
@@ -836,38 +818,56 @@ impl<D, NT, PL> PersistableOrderProtocol<D, PBFTConsensus<D>, PBFTConsensus<D>>
         Proof::new(metadata, pre_prepares, prepares, commits)
     }
 
-    fn init_dec_log(proofs: Vec<SerProof<D, PBFTConsensus<D>>>) -> DecLog<D, PBFTConsensus<D>, PBFTConsensus<D>> {
-        DecisionLog::from_proofs(proofs)
+    fn init_proof_from_scm(metadata: DecisionMetadata<D, Self::Serialization>,
+                           messages: Vec<ShareableConsensusMessage<D, Self::Serialization>>) -> PProof<D, Self::Serialization, Self::PersistableTypes> {
+        let mut pre_prepares = Vec::with_capacity(messages.len() / 2);
+        let mut prepares = Vec::with_capacity(messages.len() / 2);
+        let mut commits = Vec::with_capacity(messages.len() / 2);
+
+        for message in messages {
+            match message.message().kind() {
+                ConsensusMessageKind::PrePrepare(_) => {
+                    pre_prepares.push(message);
+                }
+                ConsensusMessageKind::Prepare(_) => {
+                    prepares.push(message);
+                }
+                ConsensusMessageKind::Commit(_) => {
+                    commits.push(message);
+                }
+            }
+        }
+
+        Proof::new(metadata, pre_prepares, prepares, commits)
     }
 
-    fn decompose_proof(proof: &SerProof<D, PBFTConsensus<D>>) -> (&SerProofMetadata<D, PBFTConsensus<D>>, Vec<&StoredMessage<LoggableMessage<D, PBFTConsensus<D>>>>) {
+    fn decompose_proof(proof: &Proof<D::Request>) -> (&ProofMetadata, Vec<&ShareableMessage<PBFTMessage<D::Request>>>) {
         let mut messages = Vec::new();
 
         for message in proof.pre_prepares() {
-            messages.push(&**message.as_ref());
+            messages.push(message);
         }
 
         for message in proof.prepares() {
-            messages.push(&**message.as_ref());
+            messages.push(message);
         }
 
         for message in proof.commits() {
-            messages.push(&**message.as_ref());
+            messages.push(message);
         }
 
         (proof.metadata(), messages)
     }
 
-    fn decompose_dec_log(proofs: &DecLog<D, PBFTConsensus<D>, PBFTConsensus<D>>) -> Vec<&SerProof<D, PBFTConsensus<D>>> {
-        proofs.proofs().iter().collect()
+    fn get_requests_in_proof(proof: &PProof<D, Self::Serialization, Self::PersistableTypes>) -> (UpdateBatch<D::Request>, Vec<ClientRqInfo>) {
+        todo!()
     }
 }
 
-impl<D, NT, PL, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, NT, PL>
+impl<D, NT, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, NT>
     where D: ApplicationData + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-          PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
     fn attempt_quorum_node_join(&mut self, joining_node: NodeId) -> Result<ReconfigurationAttemptResult> {
         let result = self.synchronizer.start_join_quorum(joining_node, &*self.node, &self.timeouts, &self.message_log);
 
diff --git a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
deleted file mode 100755
index 50bb3224..00000000
--- a/febft-pbft-consensus/src/bft/msg_log/decided_log/mod.rs
+++ /dev/null
@@ -1,275 +0,0 @@
-use log::error;
-
-use atlas_common::error::*;
-use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_core::ordering_protocol::{DecisionInformation, ProtocolConsensusDecision};
-use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog, StatefulOrderingProtocolLog};
-use atlas_smr_application::app::UpdateBatch;
-use atlas_smr_application::serialize::ApplicationData;
-
-use crate::bft::message::ConsensusMessageKind;
-use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::msg_log::deciding_log::CompletedBatch;
-use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata, StoredConsensusMessage};
-use crate::bft::msg_log::operation_key;
-use crate::bft::sync::view::ViewInfo;
-
-/// The log of decisions that have already been processed by the consensus
-/// algorithm
-pub struct Log<D, PL> where D: ApplicationData + 'static {
-    // The log for all of the already decided consensus instances
-    dec_log: DecisionLog<D::Request>,
-
-    // A handle to the persistent log
-    persistent_log: PL,
-}
-
-impl<D, PL> Log<D, PL> where D: ApplicationData + 'static {
-    pub(crate) fn init_decided_log(node_id: NodeId, persistent_log: PL,
-                                   dec_log: Option<DecisionLog<D::Request>>) -> Self {
-        Self {
-            dec_log: dec_log.unwrap_or(DecisionLog::new()),
-            persistent_log,
-        }
-    }
-
-    /// Returns a reference to a subset of this log, containing only
-    /// consensus messages.
-    pub fn decision_log(&self) -> &DecisionLog<D::Request> {
-        &self.dec_log
-    }
-
-    pub fn mut_decision_log(&mut self) -> &mut DecisionLog<D::Request> {
-        &mut self.dec_log
-    }
-
-    /// Read the current state, if existent, from the persistent storage
-    ///
-    /// FIXME: The view initialization might have to be changed if we want to introduce reconfiguration
-    pub fn read_current_state(&self, n: usize, f: usize) -> Result<Option<(ViewInfo, DecisionLog<D::Request>)>>
-        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>, PBFTConsensus<D>> {
-        let option = self.persistent_log.read_state(OperationMode::BlockingSync)?;
-
-        if let Some((view, dec_log)) = option {
-            Ok(Some((view, dec_log)))
-        } else {
-            Ok(None)
-        }
-    }
-
-    /// Take a snapshot of the log, used to recover a replica.
-    ///
-    /// This method may fail if we are waiting for the latest application
-    /// state to be returned by the execution layer.
-    ///
-    pub fn snapshot(&self, view: ViewInfo) -> Result<(ViewInfo, DecisionLog<D::Request>)> {
-        Ok((view, self.dec_log.clone()))
-    }
-
-    /// Insert a consensus message into the log.
-    /// We can use this method when we want to prevent a clone, as this takes
-    /// just a reference.
-    /// This is mostly used for pre prepares as they contain all the requests and are therefore very expensive to send
-    pub fn insert_consensus(
-        &mut self,
-        consensus_msg: StoredConsensusMessage<D::Request>,
-    ) where PL: OrderingProtocolLog<D, PBFTConsensus<D>>,
-    {
-        if let Err(err) = self
-            .persistent_log
-            .write_message(OperationMode::NonBlockingSync(None), consensus_msg)
-        {
-            error!("Failed to persist message {:?}", err);
-        }
-    }
-
-    /// Install a proof of a consensus instance into the log.
-    /// This is done when we receive the final SYNC message from the leader
-    /// which contains all of the collects
-    /// If we are missing the request determined by the
-    pub fn install_proof(&mut self, seq: SeqNo, proof: Proof<D::Request>) -> Result<ProtocolConsensusDecision<D::Request>>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        let batch_execution_info = ProtocolConsensusDecision::from(&proof);
-
-        if let Some(decision) = self.decision_log().last_decision() {
-            if decision.seq_no() == seq {
-                // Well well well, if it isn't what I'm trying to add?
-                //This should not be possible
-
-                return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidedLog,
-                                                  "Already have decision at that seq no"));
-            } else {
-                self.mut_decision_log().append_proof(proof.clone());
-            }
-        }
-
-        if let Err(err) = self.persistent_log
-            .write_proof(OperationMode::NonBlockingSync(None), proof) {
-            error!("Failed to persist proof {:?}", err);
-        }
-
-        Ok(batch_execution_info)
-    }
-
-    /// Clear the occurrences of a seq no from the decision log
-    pub fn clear_last_occurrence(&mut self, seq: SeqNo)
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        if let Err(err) = self.persistent_log.write_invalidate(OperationMode::NonBlockingSync(None), seq) {
-            error!("Failed to invalidate last occurrence {:?}", err);
-        }
-    }
-
-    /// Update the log state, received from the CST protocol.
-    pub fn install_state(&mut self, view: ViewInfo, dec_log: DecisionLog<D::Request>)
-        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>, PBFTConsensus<D>> {
-
-        //Replace the log
-        self.dec_log = dec_log.clone();
-
-        let last_seq = self.dec_log.last_execution().unwrap_or(SeqNo::ZERO);
-
-        if let Err(err) = self.persistent_log
-            .write_install_state(OperationMode::NonBlockingSync(None), view, dec_log) {
-            error!("Failed to persist message {:?}", err);
-        }
-    }
-
-    /// End the state of an on-going checkpoint.
-    ///
-    /// This method should only be called when `finalize_request()` reports
-    /// `Info::BeginCheckpoint`, and the requested application state is received
-    /// on the core server task's master channel.
-    pub fn finalize_checkpoint(&mut self, final_seq: SeqNo) -> Result<()> {
-        let mut decided_request_count;
-
-        //Clear the log of messages up to final_seq.
-        //Messages ahead of final_seq will not be removed as they are not included in the
-        //Checkpoint and therefore must be logged.
-        {
-            let mut guard = &mut self.dec_log;
-
-            decided_request_count = guard.clear_until_seq(final_seq);
-        }
-
-        Ok(())
-    }
-
-    /// Register that all the batches for a given decision have already been received
-    /// Basically persists the metadata for a given consensus num
-    pub fn all_batches_received(&mut self, metadata: ProofMetadata)
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
-        self.persistent_log.write_proof_metadata(OperationMode::NonBlockingSync(None),
-                                                 metadata).unwrap();
-    }
-
-    /// Finalize a batch of client requests decided on the consensus instance
-    /// with sequence number `seq`, retrieving the payload associated with their
-    /// given digests `digests`.
-    ///
-    /// The decided log may be cleared resulting from this operation. Check the enum variant of
-    /// `Info`, to perform a local checkpoint when appropriate.
-    ///
-    /// Returns a [`Option::None`] when we are running in Strict mode, indicating the
-    /// batch request has been put in the execution queue, waiting for all of the messages
-    /// to be persisted
-    pub fn finalize_batch(
-        &mut self,
-        seq: SeqNo,
-        completed_batch: CompletedBatch<D::Request>,
-    ) -> Result<ProtocolConsensusDecision<D::Request>> {
-        //println!("Finalized batch of OPS seq {:?} on Node {:?}", seq, self.node_id);
-
-        let batch = {
-            let mut batch = UpdateBatch::new_with_cap(seq, completed_batch.request_count());
-
-            for message in completed_batch.pre_prepare_messages() {
-                let reqs = {
-                    if let ConsensusMessageKind::PrePrepare(reqs) = (*message.message().kind()).clone() {
-                        reqs
-                    } else { unreachable!() }
-                };
-
-                for (header, message) in reqs.into_iter()
-                    .map(|x| x.into_inner()) {
-                    let _key = operation_key::<D::Request>(&header, &message);
-
-                    //TODO: Maybe make this run on separate thread?
-                    // let seq_no = latest_op_guard
-                    //     .get(key)
-                    //     .unwrap_or(&SeqNo::ZERO);
-                    //
-                    // if message.sequence_number() > *seq_no {
-                    //     latest_op_guard.insert(key, message.sequence_number());
-                    // }
-
-                    batch.add(
-                        header.from(),
-                        message.session_id(),
-                        message.sequence_number(),
-                        message.into_inner_operation(),
-                    );
-                }
-            }
-
-            batch.append_batch_meta(completed_batch.batch_meta().clone());
-
-            batch
-        };
-
-        // the last executed sequence number
-        let f = 1;
-
-        // Finalize the execution and store the proof in the log as a proof
-        // instead of an ongoing decision
-        self.dec_log.finished_quorum_execution(&completed_batch, seq, f)?;
-
-        let decision = ProtocolConsensusDecision::new(seq, batch,
-                                                      Some(DecisionInformation::from(completed_batch)));
-
-        Ok(decision)
-    }
-
-    /// Collects the most up to date data we have in store.
-    /// Accepts the f for the view that it is looking for
-    /// It must accept this f as the reconfiguration of the network
-    /// can alter the f from one seq no to the next
-    pub fn last_proof(&self, f: usize) -> Option<Proof<D::Request>> {
-        self.dec_log.last_decision()
-    }
-}
-
-impl<O> From<&Proof<O>> for ProtocolConsensusDecision<O> where O: Clone {
-    fn from(value: &Proof<O>) -> Self {
-        let mut update_batch = UpdateBatch::new(value.seq_no());
-
-        if !value.are_pre_prepares_ordered().unwrap() {
-            //The batch should be provided to this already ordered.
-            todo!()
-        }
-
-        for pre_prepare in value.pre_prepares() {
-            let consensus_msg = (*pre_prepare.message()).clone();
-
-            let reqs = match consensus_msg.into_kind() {
-                ConsensusMessageKind::PrePrepare(reqs) => { reqs }
-                _ => {
-                    unreachable!()
-                }
-            };
-
-            for request in reqs {
-                let (header, message) = request.into_inner();
-
-                update_batch.add(header.from(),
-                                 message.session_id(),
-                                 message.sequence_number(),
-                                 message.into_inner_operation());
-            }
-        }
-
-        ProtocolConsensusDecision::new(value.seq_no(),
-                                       update_batch,
-                                       None)
-    }
-}
\ No newline at end of file
diff --git a/febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs b/febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs
deleted file mode 100644
index cb01f717..00000000
--- a/febft-pbft-consensus/src/bft/msg_log/deciding_log/mod.rs
+++ /dev/null
@@ -1,588 +0,0 @@
-use std::cmp::Ordering;
-use std::collections::{BTreeMap, BTreeSet};
-use std::iter;
-use std::iter::zip;
-use std::sync::{Arc, Mutex};
-use std::time::Instant;
-
-use atlas_common::crypto::hash::{Context, Digest};
-use atlas_common::error::*;
-use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_core::messages::ClientRqInfo;
-use atlas_core::ordering_protocol::DecisionInformation;
-use atlas_metrics::benchmarks::BatchMeta;
-use atlas_metrics::metrics::metric_duration;
-
-use crate::bft::message::ConsensusMessageKind;
-use crate::bft::metric::PRE_PREPARE_LOG_ANALYSIS_ID;
-use crate::bft::msg_log::decisions::{IncompleteProof, PrepareSet, Proof, ProofMetadata, StoredConsensusMessage, ViewDecisionPair};
-use crate::bft::sync::view::ViewInfo;
-
-/// A batch that has been decided by the consensus instance and is now ready to be delivered to the
-/// Executor for execution.
-/// Contains all of the necessary information for when we are using the strict persistency mode
-pub struct CompletedBatch<O> {
-    // The sequence number of the batch
-    seq_no: SeqNo,
-
-    //The digest of the batch
-    batch_digest: Digest,
-    // The ordering of the pre prepares
-    pre_prepare_ordering: Vec<Digest>,
-    // The prepare message of the batch
-    pre_prepare_messages: Vec<StoredConsensusMessage<O>>,
-    // The prepare messages for this batch
-    prepare_messages: Vec<StoredConsensusMessage<O>>,
-    // The commit messages for this batch
-    commit_messages: Vec<StoredConsensusMessage<O>>,
-    // The information of the client requests that are contained in this batch
-    client_requests: Vec<ClientRqInfo>,
-    //The messages that must be persisted for this consensus decision to be executable
-    //This should contain the pre prepare, quorum of prepares and quorum of commits
-    messages_to_persist: Vec<Digest>,
-
-    // The metadata for this batch (mostly statistics)
-    batch_meta: BatchMeta,
-}
-
-pub struct DecidingLog<O> {
-    node_id: NodeId,
-    seq_no: SeqNo,
-
-    // Detect duplicate requests sent by replicas
-    duplicate_detection: DuplicateReplicaEvaluator,
-    //The digest of the entire batch that is currently being processed
-    // This will only be calculated when we receive all of the requests necessary
-    // As this digest requires the knowledge of all of them
-    current_digest: Option<Digest>,
-    // How many pre prepares have we received
-    current_received_pre_prepares: usize,
-    //The size of batch that is currently being processed. Increases as we receive more pre prepares
-    current_batch_size: usize,
-    //The client requests that are currently being processed
-    //Does not have to follow the correct order, only has to contain the requests
-    client_rqs: Vec<ClientRqInfo>,
-
-    // The message log of the current ongoing decision
-    ongoing_decision: OnGoingDecision<O>,
-
-    // The set of leaders that is currently in vigour for this consensus decision
-    leader_set: Vec<NodeId>,
-    // Which hash space should each leader be responsible for
-    request_space_slices: BTreeMap<NodeId, (Vec<u8>, Vec<u8>)>,
-
-    //A list of digests of all consensus related messages pertaining to this
-    //Consensus instance. Used to keep track of if the persistent log has saved the messages already
-    //So the requests can be executed
-    current_messages_to_persist: Vec<Digest>,
-
-    // Some logging information about metadata
-    batch_meta: Arc<Mutex<BatchMeta>>,
-}
-
-/// Checks to make sure replicas aren't providing more than one vote for the
-/// Same consensus decision
-#[derive(Default)]
-pub struct DuplicateReplicaEvaluator {
-    // The set of leaders that is currently in vigour for this consensus decision
-    leader_set: Vec<NodeId>,
-    // The set of leaders that have already sent a pre prepare message
-    received_pre_prepare_messages: BTreeSet<NodeId>,
-    // The set of leaders that have already sent a prepare message
-    received_prepare_messages: BTreeSet<NodeId>,
-    // The set of leaders that have already sent a commit message
-    received_commit_messages: BTreeSet<NodeId>,
-}
-
-/// Store the messages corresponding to a given ongoing consensus decision
-pub struct OnGoingDecision<O> {
-    pre_prepare_digests: Vec<Option<Digest>>,
-    // This must be a vec of options since we want to keep the ordering of the pre prepare messages
-    // Even when the messages have not yet been received
-    pre_prepare_messages: Vec<Option<StoredConsensusMessage<O>>>,
-    prepare_messages: Vec<StoredConsensusMessage<O>>,
-    commit_messages: Vec<StoredConsensusMessage<O>>,
-}
-
-/// Information about a full batch
-pub type FullBatch = ProofMetadata;
-
-impl<O> DecidingLog<O> {
-    pub fn new(node_id: NodeId, seq_no: SeqNo, view: &ViewInfo) -> Self {
-        Self {
-            node_id,
-            seq_no,
-            duplicate_detection: Default::default(),
-            current_digest: None,
-            current_received_pre_prepares: 0,
-            ongoing_decision: OnGoingDecision::initialize(view.leader_set().len()),
-            leader_set: view.leader_set().clone(),
-            request_space_slices: view.hash_space_division().clone(),
-            current_batch_size: 0,
-            current_messages_to_persist: vec![],
-            batch_meta: Arc::new(Mutex::new(BatchMeta::new())),
-            client_rqs: vec![],
-        }
-    }
-
-    /// Update our log to reflect the new views
-    pub fn update_current_view(&mut self, view: &ViewInfo) {
-        self.leader_set = view.leader_set().clone();
-        self.request_space_slices = view.hash_space_division().clone();
-    }
-
-    /// Getter for batch_meta
-    pub fn batch_meta(&self) -> &Arc<Mutex<BatchMeta>> {
-        &self.batch_meta
-    }
-
-    /// Getter for the current digest
-    pub fn current_digest(&self) -> Option<Digest> {
-        self.current_digest.clone()
-    }
-
-    /// Get the current batch size in this consensus decision
-    pub fn current_batch_size(&self) -> usize { self.current_batch_size }
-
-    /// Inform the log that we are now processing a new batch of operations
-    pub fn process_pre_prepare(&mut self,
-                               request_batch: StoredConsensusMessage<O>,
-                               digest: Digest,
-                               mut batch_rq_digests: Vec<ClientRqInfo>) -> Result<Option<FullBatch>> {
-        let start = Instant::now();
-
-        let sending_leader = request_batch.header().from();
-
-        let slice = self.request_space_slices.get(&sending_leader)
-            .ok_or(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                          format!("Failed to get request space for leader {:?}. Len: {:?}. {:?}",
-                                                  sending_leader,
-                                                  self.request_space_slices.len(),
-                                                  self.request_space_slices).as_str()))?;
-
-        if request_batch.header().from() != self.node_id {
-            for request in &batch_rq_digests {
-                if !crate::bft::sync::view::is_request_in_hash_space(&request.digest(), slice) {
-                    return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                                      "This batch contains requests that are not in the hash space of the leader."));
-                }
-            }
-        }
-
-        self.duplicate_detection.insert_pre_prepare_received(sending_leader)?;
-
-        // Get the correct index for this batch
-        let leader_index = pre_prepare_index_of(&self.leader_set, &sending_leader)?;
-
-        self.ongoing_decision.insert_pre_prepare(leader_index, request_batch.clone());
-
-        self.current_received_pre_prepares += 1;
-
-        //
-        self.current_batch_size += batch_rq_digests.len();
-
-        self.client_rqs.append(&mut batch_rq_digests);
-
-        // Register this new batch as one that must be persisted for this batch to be executed
-        self.register_message_to_save(digest);
-
-        metric_duration(PRE_PREPARE_LOG_ANALYSIS_ID, start.elapsed());
-
-        // if we have received all of the messages in the set, calculate the digest.
-        Ok(if self.current_received_pre_prepares == self.leader_set.len() {
-            // We have received all of the required batches
-            let result = self.calculate_instance_digest();
-
-            let (digest, ordering) = result
-                .ok_or(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog, "Failed to calculate instance digest"))?;
-
-            self.current_digest = Some(digest.clone());
-
-            Some(ProofMetadata::new(self.seq_no, digest, ordering))
-        } else {
-            None
-        })
-    }
-
-    /// Calculate the instance of a completed consensus pre prepare phase with
-    /// all the batches received
-    fn calculate_instance_digest(&self) -> Option<(Digest, Vec<Digest>)> {
-        let mut ctx = Context::new();
-
-        let mut batch_ordered_digests = Vec::with_capacity(self.ongoing_decision.pre_prepare_digests.len());
-
-        for order_digest in &self.ongoing_decision.pre_prepare_digests {
-            if let Some(digest) = order_digest.clone() {
-                ctx.update(digest.as_ref());
-                batch_ordered_digests.push(digest);
-            } else {
-                return None;
-            }
-        }
-
-        Some((ctx.finish(), batch_ordered_digests))
-    }
-
-    /// Process the message received
-    pub(crate) fn process_message(&mut self, message: StoredConsensusMessage<O>) -> Result<()> {
-        match message.message().kind() {
-            ConsensusMessageKind::Prepare(_) => {
-                self.duplicate_detection.insert_prepare_received(message.header().from())?;
-            }
-            ConsensusMessageKind::Commit(_) => {
-                self.duplicate_detection.insert_commit_received(message.header().from())?;
-            }
-            _ => unreachable!()
-        }
-
-        self.ongoing_decision.insert_message(message);
-
-        Ok(())
-    }
-
-    /// Get the current decision
-    pub fn deciding(&self, f: usize) -> IncompleteProof {
-        let in_exec = self.seq_no;
-
-        self.ongoing_decision.deciding(in_exec, f)
-    }
-
-    /// Indicate that the batch is finished processing and
-    /// return the relevant information for it
-    pub fn finish_processing_batch(self) -> Option<CompletedBatch<O>> {
-        let new_meta = BatchMeta::new();
-        let batch_meta = std::mem::replace(&mut *self.batch_meta().lock().unwrap(), new_meta);
-
-        let OnGoingDecision {
-            pre_prepare_digests,
-            pre_prepare_messages,
-            prepare_messages,
-            commit_messages
-        } = self.ongoing_decision;
-
-        let current_digest = self.current_digest?;
-
-        let pre_prepare_ordering = pre_prepare_digests.into_iter().map(|elem| elem.unwrap()).collect();
-        let pre_prepare_messages = pre_prepare_messages.into_iter().map(|elem| elem.unwrap()).collect();
-
-        let messages_to_persist = self.current_messages_to_persist;
-
-        Some(CompletedBatch {
-            batch_digest: current_digest,
-            seq_no: self.seq_no,
-            pre_prepare_ordering,
-            pre_prepare_messages,
-            prepare_messages,
-            commit_messages,
-            client_requests: self.client_rqs,
-            messages_to_persist,
-            batch_meta,
-        })
-    }
-
-    fn register_message_to_save(&mut self, message: Digest) {
-        self.current_messages_to_persist.push(message);
-    }
-}
-
-impl<O> OnGoingDecision<O> {
-    pub fn initialize(leader_count: usize) -> Self {
-        Self {
-            pre_prepare_digests: iter::repeat(None).take(leader_count).collect(),
-            pre_prepare_messages: iter::repeat(None).take(leader_count).collect(),
-            prepare_messages: Vec::new(),
-            commit_messages: Vec::new(),
-        }
-    }
-
-    pub fn initialize_with_ordering(leader_count: usize, ordering: Vec<Digest>) -> Self {
-        Self {
-            pre_prepare_digests: ordering.into_iter().map(|d| Some(d)).collect(),
-            pre_prepare_messages: iter::repeat(None).take(leader_count).collect(),
-            prepare_messages: Vec::new(),
-            commit_messages: Vec::new(),
-        }
-    }
-
-    /// Insert a pre prepare message into this on going decision
-    fn insert_pre_prepare(&mut self, index: usize, pre_prepare: StoredConsensusMessage<O>) {
-        if index >= self.pre_prepare_messages.len() {
-            unreachable!("Cannot insert a pre prepare message that was sent by a leader that is out of bounds")
-        }
-
-        self.pre_prepare_digests[index] = Some(pre_prepare.header().digest().clone());
-        self.pre_prepare_messages[index] = Some(pre_prepare);
-    }
-
-    /// Insert a consensus message into this on going decision
-    fn insert_message(&mut self, message: StoredConsensusMessage<O>) {
-        match message.message().kind() {
-            ConsensusMessageKind::Prepare(_) => {
-                self.prepare_messages.push(message);
-            }
-            ConsensusMessageKind::Commit(_) => {
-                self.commit_messages.push(message);
-            }
-            _ => {
-                unreachable!("Please use insert_pre_prepare to insert a pre prepare message")
-            }
-        }
-    }
-
-    /// Insert a message from the stored message into this on going decision
-    pub fn insert_persisted_msg(&mut self, message: StoredConsensusMessage<O>) -> Result<()> {
-        match message.message().kind() {
-            ConsensusMessageKind::PrePrepare(_) => {
-                let index = pre_prepare_index_from_digest_opt(&self.pre_prepare_digests, message.header().digest())?;
-
-                self.pre_prepare_messages[index] = Some(message);
-            }
-            _ => {
-                self.insert_message(message);
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Get the current decision
-    pub fn deciding(&self, in_exec: SeqNo, f: usize) -> IncompleteProof {
-
-        // fetch write set
-        let write_set = PrepareSet({
-            let mut buf = Vec::new();
-
-            for stored in self.prepare_messages.iter().rev() {
-                match stored.message().sequence_number().cmp(&in_exec) {
-                    Ordering::Equal => {
-                        let digest = match stored.message().kind() {
-                            ConsensusMessageKind::Prepare(d) => d.clone(),
-                            _ => unreachable!(),
-                        };
-
-                        buf.push(ViewDecisionPair(
-                            stored.message().view(),
-                            digest,
-                        ));
-                    }
-                    Ordering::Less => break,
-                    // impossible, because we are executing `in_exec`
-                    Ordering::Greater => unreachable!(),
-                }
-            }
-
-            buf
-        });
-
-        // fetch quorum prepares
-        let quorum_prepares = 'outer: loop {
-            // NOTE: check `last_decision` comment on quorum
-            let quorum = f << 1;
-            let mut last_view = None;
-            let mut count = 0;
-
-            for stored in self.prepare_messages.iter().rev() {
-                match stored.message().sequence_number().cmp(&in_exec) {
-                    Ordering::Equal => {
-                        match last_view {
-                            None => (),
-                            Some(v) if stored.message().view() == v => (),
-                            _ => count = 0,
-                        }
-                        last_view = Some(stored.message().view());
-                        count += 1;
-                        if count == quorum {
-                            let digest = match stored.message().kind() {
-                                ConsensusMessageKind::Prepare(d) => d.clone(),
-                                _ => unreachable!(),
-                            };
-                            break 'outer Some(ViewDecisionPair(stored.message().view(), digest));
-                        }
-                    }
-                    Ordering::Less => break,
-                    // impossible, because we are executing `in_exec`
-                    Ordering::Greater => unreachable!(),
-                }
-            }
-
-            break 'outer None;
-        };
-
-        IncompleteProof::new(in_exec, write_set, quorum_prepares)
-    }
-}
-
-impl<O> Orderable for DecidingLog<O> {
-    fn sequence_number(&self) -> SeqNo {
-        self.seq_no
-    }
-}
-
-impl DuplicateReplicaEvaluator {
-    fn insert_pre_prepare_received(&mut self, node_id: NodeId) -> Result<()> {
-        if !self.received_pre_prepare_messages.insert(node_id) {
-            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                              "We have already received a message from that leader."));
-        }
-
-        Ok(())
-    }
-    fn insert_prepare_received(&mut self, node_id: NodeId) -> Result<()> {
-        if !self.received_prepare_messages.insert(node_id) {
-            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                              "We have already received a message from that leader."));
-        }
-
-        Ok(())
-    }
-    fn insert_commit_received(&mut self, node_id: NodeId) -> Result<()> {
-        if !self.received_commit_messages.insert(node_id) {
-            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                              "We have already received a message from that leader."));
-        }
-
-        Ok(())
-    }
-}
-
-impl<O> Orderable for CompletedBatch<O> {
-    fn sequence_number(&self) -> SeqNo {
-        self.seq_no
-    }
-}
-
-impl<O> CompletedBatch<O> {
-    pub fn batch_digest(&self) -> Digest {
-        self.batch_digest
-    }
-    pub fn pre_prepare_ordering(&self) -> &Vec<Digest> {
-        &self.pre_prepare_ordering
-    }
-    pub fn pre_prepare_messages(&self) -> &Vec<StoredConsensusMessage<O>> {
-        &self.pre_prepare_messages
-    }
-
-    pub fn messages_to_persist(&self) -> &Vec<Digest> {
-        &self.messages_to_persist
-    }
-    pub fn batch_meta(&self) -> &BatchMeta {
-        &self.batch_meta
-    }
-
-    pub fn request_count(&self) -> usize {
-        self.client_requests.len()
-    }
-
-    /// Create a proof from the completed batch
-    pub fn proof(&self, quorum: Option<usize>) -> Result<Proof<O>> {
-        let (leader_count, order) = (self.pre_prepare_ordering.len(), &self.pre_prepare_ordering);
-
-        if self.pre_prepare_messages.len() != leader_count {
-            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecisions,
-                                              format!("Failed to create a proof, pre_prepares do not match up to the leader count {} leader count {} preprepares",
-                                                      leader_count, self.prepare_messages.len()).as_str()));
-        }
-
-        let mut ordered_pre_prepares = Vec::with_capacity(leader_count);
-
-        for digest in order {
-            for i in 0..self.pre_prepare_messages.len() {
-                let message = &self.pre_prepare_messages[i];
-
-                if *message.header().digest() == *digest {
-                    ordered_pre_prepares.push(self.pre_prepare_messages[i].clone());
-
-                    break;
-                }
-            }
-        }
-
-        if let Some(quorum) = quorum {
-            if self.prepare_messages.len() < quorum {
-                return Err(Error::simple_with_msg(ErrorKind::MsgLogDecisions,
-                                                  "Failed to create a proof, prepares do not match up to the 2*f+1"));
-            }
-
-            if self.commit_messages.len() < quorum {
-                return Err(Error::simple_with_msg(ErrorKind::MsgLogDecisions,
-                                                  "Failed to create a proof, commits do not match up to the 2*f+1"));
-            }
-        }
-
-        let metadata = ProofMetadata::new(self.seq_no,
-                                          self.batch_digest.clone(),
-                                          order.clone());
-
-        Ok(Proof::new(metadata,
-                      ordered_pre_prepares,
-                      self.prepare_messages.clone(),
-                      self.commit_messages.clone()))
-    }
-}
-
-pub fn pre_prepare_index_from_digest_opt(prepare_set: &Vec<Option<Digest>>, digest: &Digest) -> Result<usize> {
-    match prepare_set.iter().position(|pre_prepare| pre_prepare.map(|d| d == *digest).unwrap_or(false)) {
-        None => {
-            Err(Error::simple_with_msg(ErrorKind::Consensus, "Pre prepare is not part of the pre prepare set"))
-        }
-        Some(pos) => {
-            Ok(pos)
-        }
-    }
-}
-
-pub fn pre_prepare_index_of_from_digest(prepare_set: &Vec<Digest>, preprepare: &Digest) -> Result<usize> {
-    match prepare_set.iter().position(|pre_prepare| *pre_prepare == *preprepare) {
-        None => {
-            Err(Error::simple_with_msg(ErrorKind::Consensus, "Pre prepare is not part of the pre prepare set"))
-        }
-        Some(pos) => {
-            Ok(pos)
-        }
-    }
-}
-
-pub fn pre_prepare_index_of(leader_set: &Vec<NodeId>, proposer: &NodeId) -> Result<usize> {
-    match leader_set.iter().position(|node| *node == *proposer) {
-        None => {
-            Err(Error::simple_with_msg(ErrorKind::Consensus, "Proposer is not part of the leader set"))
-        }
-        Some(pos) => {
-            Ok(pos)
-        }
-    }
-}
-
-pub fn make_proof_from<O>(proof_meta: ProofMetadata, mut ongoing: OnGoingDecision<O>) -> Result<Proof<O>> {
-    let OnGoingDecision {
-        pre_prepare_digests,
-        pre_prepare_messages,
-        prepare_messages,
-        commit_messages
-    } = ongoing;
-
-    let pre_prepare_messages: Vec<StoredConsensusMessage<O>> = pre_prepare_messages.into_iter()
-        .map(|elem| {
-            elem.unwrap()
-        }).collect();
-
-    for (digest, digest2) in zip(proof_meta.pre_prepare_ordering(), &pre_prepare_digests) {
-        let digest2 = digest2.as_ref().ok_or(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog, "Failed to create a proof, pre prepare messages are missing"))?;
-
-        if digest != digest2 {
-            return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidingLog,
-                                              "Failed to create a proof, pre prepares do not match up to the ordering"));
-        }
-    }
-
-    Ok(Proof::new(proof_meta, pre_prepare_messages, prepare_messages, commit_messages))
-}
-
-impl<O> From<CompletedBatch<O>> for DecisionInformation {
-    fn from(value: CompletedBatch<O>) -> Self {
-        DecisionInformation::new(value.batch_digest,
-                                 value.messages_to_persist,
-                                 value.client_requests)
-    }
-}
\ No newline at end of file
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index b6cfb98a..ab8ba438 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -24,19 +24,20 @@ use atlas_communication::message::{Header, StoredMessage, WireMessage};
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_core::ordering_protocol::ProtocolConsensusDecision;
+use atlas_core::ordering_protocol::{Decision, ProtocolConsensusDecision};
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::ReconfigurationAttemptResult;
-use atlas_core::persistent_log::{OrderingProtocolLog, StatefulOrderingProtocolLog};
+use atlas_core::persistent_log::{OrderingProtocolLog};
 use atlas_core::request_pre_processing::RequestPreProcessor;
+use atlas_core::smr::smr_decision_log::ShareableMessage;
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_smr_application::serialize::ApplicationData;
 
-use crate::bft::consensus::Consensus;
+use crate::bft::consensus::{Consensus, ConsensusStatus};
+use crate::bft::log::decisions::{CollectData, Proof, ProofMetadata, ViewDecisionPair};
+use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, FwdConsensusMessage, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::{CollectData, Proof, StoredConsensusMessage, ViewDecisionPair};
-use crate::bft::PBFT;
+use crate::bft::{PBFT, PBFTOrderProtocol};
 use crate::bft::sync::view::ViewInfo;
 
 use self::{follower_sync::FollowerSynchronizer, replica_sync::ReplicaSynchronizer};
@@ -121,7 +122,6 @@ pub struct LeaderCollects<O> {
 }
 
 impl<O> LeaderCollects<O> {
-
     pub fn message(&self) -> &FwdConsensusMessage<O> {
         &self.proposed
     }
@@ -189,11 +189,11 @@ pub struct TboQueue<O> {
     // fetching them from the network
     get_queue: bool,
     // stores all STOP messages for the next view
-    stop: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O>>>>,
+    stop: VecDeque<VecDeque<ShareableMessage<PBFTMessage<O>>>>,
     // stores all STOP-DATA messages for the next view
-    stop_data: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O>>>>,
+    stop_data: VecDeque<VecDeque<ShareableMessage<PBFTMessage<O>>>>,
     // stores all SYNC messages for the next view
-    sync: VecDeque<VecDeque<StoredMessage<ViewChangeMessage<O>>>>,
+    sync: VecDeque<VecDeque<ShareableMessage<PBFTMessage<O>>>>,
 }
 
 impl<O> TboQueue<O> {
@@ -270,11 +270,11 @@ impl<O> TboQueue<O> {
 
     /// Queues a view change message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    pub fn queue(&mut self, h: Header, m: ViewChangeMessage<O>) {
-        match m.kind() {
-            ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_) => self.queue_stop(h, m),
-            ViewChangeMessageKind::StopData(_) => self.queue_stop_data(h, m),
-            ViewChangeMessageKind::Sync(_) => self.queue_sync(h, m),
+    pub fn queue(&mut self, m: ShareableMessage<PBFTMessage<O>>) {
+        match m.message().view_change().kind() {
+            ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_) => self.queue_stop(m),
+            ViewChangeMessageKind::StopData(_) => self.queue_stop_data(m),
+            ViewChangeMessageKind::Sync(_) => self.queue_sync(m),
         }
     }
 
@@ -298,25 +298,25 @@ impl<O> TboQueue<O> {
 
     /// Queues a `STOP` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    fn queue_stop(&mut self, h: Header, m: ViewChangeMessage<O>) {
+    fn queue_stop(&mut self, m: ShareableMessage<PBFTMessage<O>>) {
         // NOTE: we use next() because we want to retrieve messages
         // for v+1, as we haven't started installing the new view yet
         let seq = self.view.sequence_number().next();
-        tbo_queue_message(seq, &mut self.stop, StoredMessage::new(h, m))
+        tbo_queue_message(seq, &mut self.stop, m)
     }
 
     /// Queues a `STOP-DATA` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    fn queue_stop_data(&mut self, h: Header, m: ViewChangeMessage<O>) {
+    fn queue_stop_data(&mut self, m: ShareableMessage<PBFTMessage<O>>) {
         let seq = self.view.sequence_number().next();
-        tbo_queue_message(seq, &mut self.stop_data, StoredMessage::new(h, m))
+        tbo_queue_message(seq, &mut self.stop_data, m)
     }
 
     /// Queues a `SYNC` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
-    fn queue_sync(&mut self, h: Header, m: ViewChangeMessage<O>) {
+    fn queue_sync(&mut self, m: ShareableMessage<PBFTMessage<O>>) {
         let seq = self.view.sequence_number().next();
-        tbo_queue_message(seq, &mut self.sync, StoredMessage::new(h, m))
+        tbo_queue_message(seq, &mut self.sync, m)
     }
 
     pub fn view(&self) -> &ViewInfo {
@@ -368,10 +368,10 @@ pub enum SynchronizerStatus<O> {
     /// The view change protocol is currently running.
     Running,
     /// The view change protocol just finished running.
-    NewView(Option<ProtocolConsensusDecision<O>>),
+    NewView(ConsensusStatus<O>, Option<Decision<ProofMetadata, PBFTMessage<O>, O>>),
     /// The view change protocol just finished running and we
     /// have successfully joined the quorum.
-    NewViewJoinedQuorum(Option<ProtocolConsensusDecision<O>>, NodeId),
+    NewViewJoinedQuorum(ConsensusStatus<O>, Option<Decision<ProofMetadata, PBFTMessage<O>, O>>, NodeId),
 
     /// Before we finish the view change protocol, we need
     /// to run the CST protocol.
@@ -395,7 +395,7 @@ pub enum SynchronizerPollStatus<O> {
     Recv,
     /// A new view change message is available to be processed, retrieved from the
     /// Ordered queue
-    NextMessage(Header, ViewChangeMessage<O>),
+    NextMessage(ShareableMessage<PBFTMessage<O>>),
     /// We need to resume the view change protocol, after
     /// running the CST protocol.
     ResumeViewChange,
@@ -428,10 +428,10 @@ pub trait AbstractSynchronizer<D> where D: ApplicationData + 'static {
     /// running the view change protocol.
     fn received_view_from_state_transfer(&self, view: ViewInfo) -> bool;
 
-    fn queue(&self, header: Header, message: ViewChangeMessage<D::Request>);
+    fn queue(&self, message: ShareableMessage<PBFTMessage<D::Request>>);
 }
 
-type CollectsType<D: ApplicationData> = IntMap<StoredMessage<ViewChangeMessage<D::Request>>>;
+type CollectsType<D: ApplicationData> = IntMap<ShareableMessage<PBFTMessage<D::Request>>>;
 
 ///The synchronizer for the SMR protocol
 /// This part of the protocol is responsible for handling the changing of views and
@@ -531,8 +531,8 @@ impl<D> AbstractSynchronizer<D> for Synchronizer<D>
         false
     }
 
-    fn queue(&self, header: Header, message: ViewChangeMessage<D::Request>) {
-        self.tbo.lock().unwrap().queue(header, message)
+    fn queue(&self, message: ShareableMessage<PBFTMessage<D::Request>>) {
+        self.tbo.lock().unwrap().queue(message)
     }
 }
 
@@ -641,8 +641,8 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                 );
 
                 match &result {
-                    SynchronizerPollStatus::NextMessage(h, vc) => {
-                        match vc.kind() {
+                    SynchronizerPollStatus::NextMessage(message) => {
+                        match message.message().view_change().kind() {
                             ViewChangeMessageKind::StopQuorumJoin(_) => {
                                 self.phase.replace(ProtoPhase::ViewStopping(0));
                             }
@@ -683,17 +683,19 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     // TODO: retransmit STOP msgs
     pub fn process_message<NT, PL>(
         &self,
-        header: Header,
-        message: ViewChangeMessage<D::Request>,
+        s_message: ShareableMessage<PBFTMessage<D::Request>>,
         timeouts: &Timeouts,
-        log: &mut Log<D, PL>,
+        log: &mut Log<D>,
         rq_pre_processor: &RequestPreProcessor<D::Request>,
-        consensus: &mut Consensus<D, PL>,
+        consensus: &mut Consensus<D>,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
               PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
+
+        let (header, message) = (s_message.header(), s_message.message().view_change());
+
         debug!("{:?} // Processing view change message {:?} in phase {:?} from {:?}",
                node.id(),
                message,
@@ -708,7 +710,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
                         debug!("{:?} // Received {:?} message while in init state. Queueing", node.id(), message);
 
-                        guard.queue_stop(header, message);
+                        guard.queue_stop(s_message);
 
                         SynchronizerStatus::Nil
                     }
@@ -722,7 +724,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                                 let mut guard = self.tbo.lock().unwrap();
                                 debug!("{:?} // Received stop data {:?} message while in init state. Queueing", node.id(), message);
 
-                                guard.queue_stop_data(header, message);
+                                guard.queue_stop_data(s_message);
 
                                 SynchronizerStatus::Nil
                             }
@@ -731,7 +733,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                     ViewChangeMessageKind::Sync(_) => {
                         let mut guard = self.tbo.lock().unwrap();
 
-                        guard.queue_sync(header, message);
+                        guard.queue_sync(s_message);
 
                         debug!("{:?} // Received sync message while in init state. Queueing", node.id());
 
@@ -750,7 +752,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
                         let mut guard = self.tbo.lock().unwrap();
 
-                        guard.queue_stop(header, message);
+                        guard.queue_stop(s_message);
 
                         return stop_status!(i, &current_view);
                     }
@@ -776,7 +778,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                                 {
                                     let mut guard = self.tbo.lock().unwrap();
 
-                                    guard.queue_stop_data(header, message);
+                                    guard.queue_stop_data(s_message);
 
                                     debug!("{:?} // Received stop data message while in stopping state. Queueing", node.id());
                                 }
@@ -789,7 +791,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                         {
                             let mut guard = self.tbo.lock().unwrap();
 
-                            guard.queue_sync(header, message);
+                            guard.queue_sync(s_message);
 
                             debug!("{:?} // Received sync message while in init state. Queueing", node.id());
                         }
@@ -869,7 +871,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
                         let mut guard = self.tbo.lock().unwrap();
 
-                        guard.queue_stop(header, message);
+                        guard.queue_stop(s_message);
 
                         return stop_status!(received, &current_view);
                     }
@@ -878,7 +880,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
                         warn!("{:?} // Received stop message while in View Stopping state. Since STOP takes precendence over the quorum updating, we will now change to stopping phase ", node.id());
 
-                        guard.queue_stop(header, message);
+                        guard.queue_stop(s_message);
 
                         self.phase.replace(ProtoPhase::Stopping(0));
 
@@ -895,7 +897,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                                 {
                                     let mut guard = self.tbo.lock().unwrap();
 
-                                    guard.queue_stop_data(header, message);
+                                    guard.queue_stop_data(s_message);
 
                                     debug!("{:?} // Received stop data message while in stopping state. Queueing", node.id());
                                 }
@@ -908,7 +910,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                         {
                             let mut guard = self.tbo.lock().unwrap();
 
-                            guard.queue_sync(header, message);
+                            guard.queue_sync(s_message);
 
                             debug!("{:?} // Received sync message while in init state. Queueing", node.id());
                         }
@@ -1010,7 +1012,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                                 {
                                     let mut guard = self.tbo.lock().unwrap();
 
-                                    guard.queue_stop(header, message);
+                                    guard.queue_stop(s_message);
 
                                     debug!("{:?} // Received stop message while in stopping data state. Queueing", node.id());
                                 }
@@ -1031,7 +1033,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                                     {
                                         let mut guard = self.tbo.lock().unwrap();
 
-                                        guard.queue_stop_data(header, message);
+                                        guard.queue_stop_data(s_message);
                                     }
                                 }
 
@@ -1063,7 +1065,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                                     let mut guard = self.tbo.lock().unwrap();
                                     //Since we are the current leader and are waiting for stop data,
                                     //This must be related to another view.
-                                    guard.queue_sync(header, message);
+                                    guard.queue_sync(s_message);
                                 }
 
                                 debug!("{:?} // Received sync message while in stopping data phase. Queueing", node.id());
@@ -1079,7 +1081,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                         // the new leader isn't forging messages.
 
                         // store collects from this STOP-DATA
-                        collects_guard.insert(header.from().into(), StoredMessage::new(header, message));
+                        collects_guard.insert(header.from().into(), s_message);
 
                         if i < next_view.params().quorum() {
                             self.phase.replace(ProtoPhase::StoppingData(i));
@@ -1221,7 +1223,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                         {
                             let mut guard = self.tbo.lock().unwrap();
 
-                            guard.queue_stop(header, message);
+                            guard.queue_stop(s_message);
 
                             debug!("{:?} // Received stop message while in syncing phase. Queueing", node.id());
                         }
@@ -1241,7 +1243,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                                 {
                                     let mut guard = self.tbo.lock().unwrap();
 
-                                    guard.queue_stop_data(header, message);
+                                    guard.queue_stop_data(s_message);
 
                                     debug!("{:?} // Received stop data message while in syncing phase. Queueing", node.id());
                                 }
@@ -1256,7 +1258,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
                             let mut guard = self.tbo.lock().unwrap();
 
-                            guard.queue_sync(header, message);
+                            guard.queue_sync(s_message);
                         }
 
                         return SynchronizerStatus::Running;
@@ -1324,17 +1326,15 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     }
 
     /// Resume the view change protocol after running the CST protocol.
-    pub fn resume_view_change<NT, PL>(
+    pub fn resume_view_change<NT>(
         &self,
-        log: &mut Log<D, PL>,
+        log: &mut Log<D>,
         timeouts: &Timeouts,
-        consensus: &mut Consensus<D, PL>,
+        consensus: &mut Consensus<D>,
         node: &Arc<NT>,
     ) -> Option<SynchronizerStatus<D::Request>>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-              PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
-
         let state = self.finalize_state.borrow_mut().take()?;
 
         //This is kept alive until it is out of the scope
@@ -1354,7 +1354,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
     /// Start the quorum join procedure to integrate the given joining node into the current quorum
     /// of the system
-    pub fn start_join_quorum<NT, PL>(&self, joining_node: NodeId, node: &NT, timeouts: &Timeouts, log: &Log<D, PL>) -> SyncReconfigurationResult
+    pub fn start_join_quorum<NT, PL>(&self, joining_node: NodeId, node: &NT, timeouts: &Timeouts, log: &Log<D>) -> SyncReconfigurationResult
         where NT: OrderProtocolSendNode<D, PBFT<D>>,
               PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
         let current_view = self.view();
@@ -1379,11 +1379,11 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                     SyncReconfigurationResult::OnGoingQuorumChange(currently_adding)
                 } else {
                     SyncReconfigurationResult::OnGoingViewChange
-                }
+                };
             }
             ProtoPhase::ViewStopping2(_) | ProtoPhase::Stopping(_) | ProtoPhase::Stopping2(_) => {
                 // Here we still don't know what is the target of the view change.
-                return SyncReconfigurationResult::OnGoingViewChange
+                return SyncReconfigurationResult::OnGoingViewChange;
             }
             _ => {
                 info!("{:?} // Attempted to add node {:?} quorum but we are currently performing a view change", node.id(), joining_node);
@@ -1451,7 +1451,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                                             join_cert: Option<NodeId>,
                                             node: &NT,
                                             timeouts: &Timeouts,
-                                            _log: &Log<D, PL>, )
+                                            _log: &Log<D>, )
         where NT: OrderProtocolSendNode<D, PBFT<D>>,
               PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
@@ -1507,7 +1507,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         timed_out: Option<Vec<StoredRequestMessage<D::Request>>>,
         node: &NT,
         timeouts: &Timeouts,
-        _log: &Log<D, PL>,
+        _log: &Log<D>,
     ) where NT: OrderProtocolSendNode<D, PBFT<D>>,
             PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
@@ -1571,14 +1571,13 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
     // this function mostly serves the purpose of consuming
 // values with immutable references, to allow borrowing data mutably
-    fn pre_finalize<PL>(
+    fn pre_finalize(
         &self,
         state: FinalizeState<D::Request>,
         proof: Option<&Proof<D::Request>>,
         _normalized_collects: Vec<Option<&CollectData<D::Request>>>,
-        log: &Log<D, PL>,
+        log: &Log<D>,
     ) -> FinalizeStatus<D::Request>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
         let last_executed_cid = proof.as_ref().map(|p| p.sequence_number()).unwrap_or(SeqNo::ZERO);
 
@@ -1604,16 +1603,15 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
     /// Finalize a view change and install the new view in the other
     /// state machines (Consensus)
-    fn finalize<NT, PL>(
+    fn finalize<NT>(
         &self,
         state: FinalizeState<D::Request>,
-        log: &mut Log<D, PL>,
+        log: &mut Log<D>,
         timeouts: &Timeouts,
-        consensus: &mut Consensus<D, PL>,
+        consensus: &mut Consensus<D>,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-              PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
         let FinalizeState {
             curr_cid,
@@ -1629,10 +1627,6 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
         warn!("{:?} // Finalizing view change to view {:?} and consensus ID {:?}, Adding node? {:?}", node.id(), view, curr_cid, self.currently_adding_node.get());
 
-        // we will get some value to be proposed because of the
-        // check we did in `pre_finalize()`, guarding against no values
-        log.clear_last_occurrence(curr_cid);
-
         let (header, message) = proposed.into_inner();
 
         let last_executed_cid = last_proof.as_ref().map(|p| p.sequence_number()).unwrap_or(SeqNo::ZERO);
@@ -1645,7 +1639,14 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             // We are missing the last decision, which should be included in the collect data
             // sent by the leader in the SYNC message
             let to_execute = if let Some(last_proof) = last_proof {
-                Some(consensus.catch_up_to_quorum(last_proof.seq_no(), &view, last_proof, log).expect("Failed to catch up to quorum"))
+                let proof = last_proof.metadata().clone();
+                let messages = last_proof.all_messages();
+                let quorum_result = consensus.catch_up_to_quorum(last_proof.seq_no(), &view, last_proof, log)
+                    .expect("Failed to catch up to quorum");
+
+                let decision = Decision::full_decision_info(last_executed_cid, proof, messages, quorum_result);
+
+                Some(decision)
             } else {
                 // This maybe happens when a checkpoint is done and the first execution after it
                 // fails, leading to a view change? Don't really know how this would be possible
@@ -1659,7 +1660,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         };
 
         // finalize view change by broadcasting a PREPARE msg
-        consensus.finalize_view_change((header, message), &view, self, timeouts, log, node);
+        let consensus_result = consensus.finalize_view_change((header, message), &view, self, timeouts, log, node)?;
 
         // Update proto phase
         self.phase.replace(ProtoPhase::Init);
@@ -1669,10 +1670,10 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
             self.currently_adding.borrow_mut().clear();
 
-            SynchronizerStatus::NewViewJoinedQuorum(to_execute, node.unwrap())
+            SynchronizerStatus::NewViewJoinedQuorum(consensus_result, to_execute, node.unwrap())
         } else {
             // resume normal phase
-            SynchronizerStatus::NewView(to_execute)
+            SynchronizerStatus::NewView(consensus_result, to_execute)
         }
     }
 
@@ -2087,8 +2088,8 @@ impl<O> Debug for SynchronizerPollStatus<O> {
             SynchronizerPollStatus::Recv => {
                 write!(f, "SynchronizerPollStatus::Recv")
             }
-            SynchronizerPollStatus::NextMessage(_, _) => {
-                write!(f, "SynchronizerPollStatus::NextMessage")
+            SynchronizerPollStatus::NextMessage(message) => {
+                write!(f, "SynchronizerPollStatus::NextMessage Header {:?}, Message {:?}", message.header(), message.message().view_change())
             }
             SynchronizerPollStatus::ResumeViewChange => {
                 write!(f, "SynchronizerPollStatus::ResumeViewChange")

From 02faccb220ccbfe57dd4459aeb5c71dc4f19b38b Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Tue, 10 Oct 2023 21:52:09 +0100
Subject: [PATCH 59/80] Log transfer adapted to not required permissioned
 protocols and to not transfer views. Started fixing the persistent log. More
 work on FeBFT and adapting it.

---
 febft-pbft-consensus/src/bft/consensus/mod.rs |  23 +-
 .../src/bft/log/deciding/mod.rs               |  60 +++-
 .../src/bft/log/decisions/mod.rs              |  50 ++--
 febft-pbft-consensus/src/bft/log/mod.rs       | 121 +++++++-
 febft-pbft-consensus/src/bft/mod.rs           | 258 +++++++++---------
 febft-pbft-consensus/src/bft/sync/mod.rs      |  15 +-
 6 files changed, 347 insertions(+), 180 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 326991d3..1c5a81b2 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -24,7 +24,7 @@ use atlas_smr_application::ExecutorHandle;
 use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_increment;
 
-use crate::bft::{PBFT, SysMsg};
+use crate::bft::{OPDecision, PBFT, SysMsg};
 use crate::bft::consensus::decision::{ConsensusDecision, DecisionPollStatus, DecisionStatus, MessageQueue};
 use crate::bft::log::deciding::CompletedBatch;
 use crate::bft::log::decisions::{DecisionLog, IncompleteProof, Proof, ProofMetadata};
@@ -48,14 +48,14 @@ pub enum ConsensusStatus<O> {
     MessageQueued,
     /// A `febft` quorum still hasn't made a decision
     /// on a client request to be executed.
-    Deciding(MaybeVec<Decision<ProofMetadata, PBFTMessage<O>, O>>),
+    Deciding(MaybeVec<OPDecision<O>>),
     /// A `febft` quorum decided on the execution of
     /// the batch of requests with the given digests.
     /// The first digest is the digest of the Prepare message
     /// And therefore the entire batch digest
     /// THe second Vec<Digest> is a vec with digests of the requests contained in the batch
     /// The third is the messages that should be persisted for this batch to be considered persisted
-    Decided(MaybeVec<Decision<ProofMetadata, PBFTMessage<O>, O>>),
+    Decided(MaybeVec<OPDecision<O>>),
 }
 
 #[derive(Debug, Clone)]
@@ -68,7 +68,7 @@ pub enum ConsensusPollStatus<O> {
     NextMessage(ShareableMessage<PBFTMessage<O>>),
     /// The first consensus instance of the consensus queue is ready to be finalized
     /// as it has already been decided
-    Decided,
+    Decided(MaybeVec<Decision<ProofMetadata, PBFTMessage<O>, O>>),
 }
 
 /// Represents a queue of messages to be ordered in a consensus instance.
@@ -190,8 +190,8 @@ pub struct Signals {
 
 /// The consensus handler. Responsible for multiplexing consensus instances and keeping track
 /// of missing messages
-pub struct Consensus<D,>
-    where D: ApplicationData + 'static,{
+pub struct Consensus<D, >
+    where D: ApplicationData + 'static, {
     node_id: NodeId,
     /// The handle to the executor of the function
     executor_handle: ExecutorHandle<D>,
@@ -355,7 +355,7 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
         // and it will be handled until completion from there, but having a backup is never
         // A bad idea
         if self.can_finalize() {
-            return ConsensusPollStatus::Decided;
+            return ConsensusPollStatus::Decided(MaybeVec::None);
         }
 
         ConsensusPollStatus::Recv
@@ -413,7 +413,7 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
 
         let decision_seq = decision.sequence_number();
 
-        let status = decision.process_message(s_message, synchronizer, timeouts,  node)?;
+        let status = decision.process_message(s_message, synchronizer, timeouts, node)?;
 
         Ok(match status {
             DecisionStatus::VotedTwice(node) => {
@@ -641,7 +641,7 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
                 while self.tbo_queue.sequence_number() < novel_seq_no && self.decisions.len() < self.watermark as usize {
                     let messages = self.tbo_queue.advance_queue();
 
-                    let decision = ConsensusDecision::init_with_msg_log(self.node_id, sequence_no, view,messages);
+                    let decision = ConsensusDecision::init_with_msg_log(self.node_id, sequence_no, view, messages);
 
                     debug!("{:?} // Initialized new decision from TBO queue messages {:?}", self.node_id, decision.sequence_number());
 
@@ -703,14 +703,13 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
 
     /// Catch up to the quorums latest decided consensus
     pub fn catch_up_to_quorum(&mut self,
-                              seq: SeqNo,
                               view: &ViewInfo,
                               proof: Proof<D::Request>,
-                              log: &mut Log<D>) -> Result<ProtocolConsensusDecision<D::Request>> {
+                              log: &mut Log<D>) -> Result<OPDecision<D::Request>> {
 
         // If this is successful, it means that we are all caught up and can now start executing the
         // batch
-        let to_execute = log.install_proof(seq, proof)?;
+        let to_execute = log.install_proof(proof)?;
 
         // Move to the next instance as this one has been finalized
         self.next_instance(view);
diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
index 46b31684..8d91ebdb 100644
--- a/febft-pbft-consensus/src/bft/log/deciding/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -8,13 +8,28 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::error::*;
 use atlas_communication::message::Header;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
+use atlas_core::ordering_protocol::networking::serialize::NetworkView;
+use atlas_core::smr::smr_decision_log::ShareableMessage;
 use atlas_metrics::benchmarks::BatchMeta;
 use atlas_metrics::metrics::metric_duration;
 use crate::bft::log::decisions::{IncompleteProof, ProofMetadata};
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::metric::PRE_PREPARE_LOG_ANALYSIS_ID;
 use crate::bft::sync::view::ViewInfo;
 
+/// The log of messages for a given batch
+pub struct MessageLog<O> {
+    pre_prepare: Vec<Option<ShareableMessage<PBFTMessage<O>>>>,
+    prepares: Vec<ShareableMessage<PBFTMessage<O>>>,
+    commits: Vec<ShareableMessage<PBFTMessage<O>>>
+}
+
+pub struct FinishedMessageLog<O> {
+    pre_prepares: Vec<ShareableMessage<PBFTMessage<O>>>,
+    prepares: Vec<ShareableMessage<PBFTMessage<O>>>,
+    commits: Vec<ShareableMessage<PBFTMessage<O>>>
+}
+
 /// Information about the completed batch, the contained requests and
 /// other relevant information
 pub struct CompletedBatch<O> {
@@ -23,10 +38,13 @@ pub struct CompletedBatch<O> {
     digest: Digest,
     // The ordering of the pre prepare requests
     pre_prepare_ordering: Vec<Digest>,
+    // The messages that are a part of this decision
+    contained_messages: FinishedMessageLog<O>,
     // The information of the client requests that are contained in this batch
     client_request_info: Vec<ClientRqInfo>,
     // The client requests contained in this batch
     client_requests: Vec<StoredRequestMessage<O>>,
+
     // The metadata for the batch
     batch_meta: BatchMeta,
 }
@@ -54,6 +72,8 @@ pub struct WorkingDecisionLog<O> {
     leader_set: Vec<NodeId>,
     // Which hash space should each leader be responsible for
     request_space_slices: BTreeMap<NodeId, (Vec<u8>, Vec<u8>)>,
+    // The log of messages of the currently working decision
+    message_log: MessageLog<O>,
     // Some logging information about metadata
     batch_meta: Arc<Mutex<BatchMeta>>,
     // The contained requests per each of the received pre prepares
@@ -74,6 +94,36 @@ pub struct DuplicateReplicaEvaluator {
     received_commit_messages: BTreeSet<NodeId>,
 }
 
+impl<O> MessageLog<O> {
+    pub fn with_leader_count(leaders: usize, quorum: usize) -> Self {
+        Self {
+            pre_prepare: iter::repeat(None).take(leaders).collect(),
+            prepares: Vec::with_capacity(quorum),
+            commits: Vec::with_capacity(quorum),
+        }
+    }
+
+    pub fn insert_pre_prepare(&mut self, index: usize, pre_prepare: ShareableMessage<PBFTMessage<O>>) {
+        let _ = self.pre_prepare.get_mut(index).unwrap().insert(pre_prepare);
+    }
+
+    pub fn insert_prepare(&mut self, prepare: ShareableMessage<PBFTMessage<O>>) {
+        self.prepares.push(prepare);
+    }
+
+    pub fn insert_commit(&mut self, commit: ShareableMessage<PBFTMessage<O>>) {
+        self.commits.push(commit);
+    }
+
+    pub fn finalize(self) -> FinishedMessageLog<O> {
+        FinishedMessageLog {
+            pre_prepares: self.pre_prepare.into_iter().map(|opt| opt.unwrap()).collect(),
+            prepares: self.prepares,
+            commits: self.commits,
+        }
+    }
+}
+
 impl<O> WorkingDecisionLog<O> where O: Clone {
     pub fn new(node: NodeId, seq: SeqNo, view: &ViewInfo) -> Self {
         let leader_count = view.leader_set().len();
@@ -88,6 +138,7 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
             client_rqs: vec![],
             leader_set: view.leader_set().clone(),
             request_space_slices: view.hash_space_division().clone(),
+            message_log: MessageLog::with_leader_count(view.leader_set().len(), view.quorum()),
             batch_meta: Arc::new(Mutex::new(BatchMeta::new())),
             contained_requests: iter::repeat(None).take(leader_count).collect(),
         }
@@ -238,6 +289,7 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
             seq: self.seq_no,
             digest: current_digest,
             pre_prepare_ordering,
+            contained_messages: self.message_log.finalize(),
             client_request_info: self.client_rqs,
             batch_meta,
             client_requests: requests
@@ -245,6 +297,12 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
     }
 }
 
+impl<O> Orderable for CompletedBatch<O> {
+    fn sequence_number(&self) -> SeqNo {
+        self.seq
+    }
+}
+
 impl<O> Orderable for WorkingDecisionLog<O> {
     fn sequence_number(&self) -> SeqNo {
         self.seq_no
diff --git a/febft-pbft-consensus/src/bft/log/decisions/mod.rs b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
index 4877750b..282308bf 100644
--- a/febft-pbft-consensus/src/bft/log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
@@ -7,14 +7,13 @@ use serde::{Deserialize, Serialize};
 
 use atlas_common::crypto::hash::Digest;
 use atlas_common::error::*;
-use atlas_common::globals::ReadOnly;
 use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::StoredMessage;
 use atlas_core::ordering_protocol::networking::serialize::{OrderProtocolLog, OrderProtocolProof};
 use atlas_core::smr::smr_decision_log::ShareableMessage;
 use crate::bft::log::deciding::CompletedBatch;
 
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
+use crate::bft::message::{ConsensusMessageKind, PBFTMessage};
 pub type StoredConsensusMessage<O> = ShareableMessage<PBFTMessage<O>>;
 
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
@@ -169,22 +168,6 @@ impl<O> Proof<O> {
         Ok(())
     }
 
-    pub fn all_messages(&self) -> Vec<StoredConsensusMessage<O>> {
-        let mut messages = Vec::with_capacity(self.pre_prepares.len() + self.prepares.len() + self.commits.len());
-
-        for pre_prepare in self.pre_prepares() {
-            messages.push(pre_prepare.clone())
-        }
-        for prepare in self.prepares() {
-            messages.push(prepare.clone())
-        }
-        for commit in self.commits() {
-            messages.push(commit.clone())
-        }
-
-        messages
-    }
-
     /// Check if the pre prepares stored here are ordered correctly according
     /// to the [pre_prepare_ordering] in this same proof.
     pub fn are_pre_prepares_ordered(&self) -> Result<bool> {
@@ -230,6 +213,25 @@ impl<O> Proof<O> {
         Ok(())
     }
 
+    pub fn into_parts(self) -> (ProofMetadata, Vec<ShareableMessage<PBFTMessage<O>>>) {
+
+        let mut vec = Vec::with_capacity(self.pre_prepares.len() + self.prepares.len() + self.commits.len());
+
+        for pre_prepares in self.pre_prepares {
+            vec.push(pre_prepares);
+        }
+
+        for prepare in self.prepares {
+            vec.push(prepare);
+        }
+
+        for commit in self.commits {
+            vec.push(commit);
+        }
+
+        (self.metadata, vec)
+    }
+
 }
 
 impl<O> Orderable for Proof<O> {
@@ -404,17 +406,7 @@ impl<O> OrderProtocolLog for DecisionLog<O> {
 
 impl<O> OrderProtocolProof for Proof<O> {
     fn contained_messages(&self) -> usize {
-        let mut messages = 0;
-
-        for pre_prepare in &self.pre_prepares {
-            //Mark the requests contained in this message for removal
-            messages += match pre_prepare.message().kind() {
-                ConsensusMessageKind::PrePrepare(messages) => messages.len(),
-                _ => 0,
-            };
-        }
-
-        messages
+        self.metadata().contained_client_rqs()
     }
 }
 
diff --git a/febft-pbft-consensus/src/bft/log/mod.rs b/febft-pbft-consensus/src/bft/log/mod.rs
index f1916a80..cda43ea9 100644
--- a/febft-pbft-consensus/src/bft/log/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/mod.rs
@@ -1,9 +1,16 @@
+use either::Either;
 use atlas_common::node_id::NodeId;
-use atlas_common::ordering::SeqNo;
+use atlas_common::ordering::{InvalidSeqNo, Orderable, SeqNo};
+use atlas_common::error::*;
 use atlas_communication::message::Header;
-use atlas_core::messages::RequestMessage;
+use atlas_core::messages::{ClientRqInfo, RequestMessage};
+use atlas_core::ordering_protocol::{Decision, ProtocolConsensusDecision};
+use atlas_smr_application::app::UpdateBatch;
 use atlas_smr_application::serialize::ApplicationData;
-use crate::bft::log::decisions::DecisionLog;
+use crate::bft::log::deciding::{CompletedBatch, FinishedMessageLog};
+use crate::bft::log::decisions::{DecisionLog, Proof, ProofMetadata};
+use crate::bft::message::ConsensusMessageKind;
+use crate::bft::OPDecision;
 
 pub mod decided;
 pub mod deciding;
@@ -14,11 +21,78 @@ pub struct Log<D> where D: ApplicationData {
 }
 
 impl<D> Log<D> where D: ApplicationData {
-
     pub fn decision_log(&self) -> &DecisionLog<D::Request> {
         &self.decided
     }
 
+    pub fn install_proof(&mut self, proof: Proof<D::Request>) -> Result<OPDecision<D::Request>> {
+        if let Some(decision) = self.decision_log().last_execution() {
+            match proof.seq_no().index(decision) {
+                Either::Left(_) | Either::Right(0) => {
+                    return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidedLog,
+                                                      "Cannot install proof as we already have a decision that is >= to the one that was provided"));
+                }
+                Either::Right(1) => {
+                    self.decided.append_proof(proof.clone());
+                }
+                Either::Right(_) => {
+                    return Err(Error::simple_with_msg(ErrorKind::MsgLogDecidedLog,
+                                                      "Cannot install proof as it would involve skipping a decision that we are not aware of"));
+                }
+            }
+        }
+
+        let batch_info = ProtocolConsensusDecision::from(&proof);
+
+        let (metadata, messages) = proof.into_parts();
+
+        Ok(Decision::full_decision_info(proof.sequence_number(), metadata, messages, batch_info))
+    }
+
+    pub fn finalize_batch(&mut self, completed: CompletedBatch<D::Request>) -> Result<ProtocolConsensusDecision<D::Request>> {
+        let CompletedBatch {
+            seq, digest,
+            pre_prepare_ordering,
+            contained_messages,
+            client_request_info,
+            client_requests,
+            batch_meta
+        } = completed;
+
+        let metadata = ProofMetadata::new(seq, digest.clone(), pre_prepare_ordering,
+                                          client_requests.len());
+
+        let FinishedMessageLog {
+            pre_prepares,
+            prepares,
+            commits
+        } = contained_messages;
+
+        let proof = Proof::new(
+            metadata,
+            pre_prepares,
+            prepares,
+            commits,
+        );
+
+        self.decided.append_proof(proof);
+
+        let mut batch = UpdateBatch::new_with_cap(seq, client_requests.len());
+
+        for cli_rq in client_requests {
+            let (header, rq) = cli_rq.into_inner();
+
+            batch.add(header.from(), rq.session_id(), rq.sequence_number(), rq.into_inner_operation());
+        }
+
+        Ok(ProtocolConsensusDecision::new(seq, batch, client_request_info, digest))
+    }
+}
+
+pub fn initialize_decided_log<D>(node_id: NodeId) -> Log<D> where D: ApplicationData {
+    Log {
+        decided: DecisionLog::new(),
+    }
 }
 
 #[inline]
@@ -35,3 +109,42 @@ pub fn operation_key_raw(from: NodeId, session: SeqNo) -> u64 {
     // therefore this is safe, and will not delete any bits
     client_id | (session_id << 32)
 }
+
+impl<O> From<&Proof<O>> for ProtocolConsensusDecision<O> where O: Clone {
+    fn from(value: &Proof<O>) -> Self {
+        let mut update_batch = UpdateBatch::new_with_cap(value.seq_no(), value.metadata().contained_client_rqs());
+        let mut client_rqs = Vec::with_capacity(value.metadata().contained_client_rqs());
+
+        if !value.are_pre_prepares_ordered().unwrap() {
+            //The batch should be provided to this already ordered.
+            todo!()
+        }
+
+        for pre_prepare in value.pre_prepares() {
+            let consensus_msg = (*pre_prepare.message()).clone();
+
+            let reqs = match consensus_msg.into_consensus().into_kind() {
+                ConsensusMessageKind::PrePrepare(reqs) => { reqs }
+                _ => {
+                    unreachable!()
+                }
+            };
+
+            for request in reqs {
+                client_rqs.push(ClientRqInfo::from(&request));
+
+                let (header, message) = request.into_inner();
+
+                update_batch.add(header.from(),
+                                 message.session_id(),
+                                 message.sequence_number(),
+                                 message.into_inner_operation());
+            }
+        }
+
+        ProtocolConsensusDecision::new(value.seq_no(),
+                                       update_batch,
+                                       client_rqs,
+                                       value.metadata().batch_digest())
+    }
+}
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 0091bdc6..b88bf3c0 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -3,12 +3,13 @@
 //! By default, it is hidden to the user, unless explicitly enabled
 //! with the feature flag `expose_impl`.
 
+use std::collections::BTreeMap;
 use std::fmt::Debug;
 use std::ops::Drop;
 use std::sync::Arc;
 use std::sync::atomic::AtomicBool;
 use std::time::Instant;
-use ::log::{debug, info, trace, warn};
+use ::log::{debug, error, info, trace, warn};
 
 use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
@@ -19,7 +20,7 @@ use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::serialize::Serializable;
 use atlas_core::messages::{ClientRqInfo, Protocol};
-use atlas_core::ordering_protocol::{Decision, DecisionInfo, DecisionMetadata, JoinInfo, OPExecResult, OPPollResult, OrderingProtocol, OrderingProtocolArgs, OrderProtocolTolerance, PermissionedOrderingProtocol, ProtocolConsensusDecision, ProtocolMessage, View};
+use atlas_core::ordering_protocol::{Decision, DecisionInfo, DecisionMetadata, DecisionsAhead, JoinInfo, OPExecResult, OPPollResult, OrderingProtocol, OrderingProtocolArgs, OrderProtocolTolerance, PermissionedOrderingProtocol, ProtocolConsensusDecision, ProtocolMessage, View};
 use atlas_core::ordering_protocol::loggable::{LoggableOrderProtocol, PProof};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::networking::serialize::{NetworkView, OrderingProtocolMessage};
@@ -37,7 +38,7 @@ use atlas_smr_application::app::UpdateBatch;
 use crate::bft::config::PBFTConfig;
 use crate::bft::consensus::{Consensus, ConsensusPollStatus, ConsensusStatus, ProposerConsensusGuard};
 use crate::bft::log::decisions::{Proof, ProofMetadata};
-use crate::bft::log::Log;
+use crate::bft::log::{initialize_decided_log, Log};
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, ObserveEventKind, PBFTMessage, ViewChangeMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{CONSENSUS_INSTALL_STATE_TIME_ID, MSG_LOG_INSTALL_TIME_ID};
@@ -72,8 +73,8 @@ pub enum ConsensusPhase {
 pub enum SyncPhaseRes<O> {
     SyncProtocolNotNeeded,
     RunSyncProtocol,
-    SyncProtocolFinished(Option<ProtocolConsensusDecision<O>>),
-    JoinedQuorum(Option<ProtocolConsensusDecision<O>>, NodeId),
+    SyncProtocolFinished(ConsensusStatus<O>, Option<OPDecision<O>>),
+    JoinedQuorum(ConsensusStatus<O>, Option<OPDecision<O>>, NodeId),
     RunCSTProtocol,
 }
 
@@ -283,7 +284,7 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
                                                     SeqNo::ZERO, watermark, consensus_guard.clone(),
                                                     timeouts.clone());
 
-        let dec_log = initialize_decided_log::<D, PL>(node_id, initial_state)?;
+        let dec_log = initialize_decided_log::<D>(node_id)?;
 
         let proposer = Proposer::<D, NT>::new(node.clone(), batch_input, sync.clone(), timeouts.clone(),
                                               executor.clone(), consensus_guard.clone(),
@@ -351,22 +352,24 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
                         SynchronizerStatus::NewViewJoinedQuorum(consensus_status, decisions, node) => {
                             let quorum_members = self.synchronizer.view().quorum_members().clone();
 
-                            let decisions;
-
-                            match consensus_status {
-                                ConsensusStatus::VotedTwice(_) | ConsensusStatus::MessageQueued | ConsensusStatus::MessageIgnored => {
-                                    decisions = None;
-                                }
-                                ConsensusStatus::Deciding(decision) | ConsensusStatus::Decided(decision) => {
-                                    decisions = Some(decision)
-                                }
-                            }
+                            let decisions = self.handle_sync_result(consensus_status, decisions)?;
 
                             let joined = JoinInfo::new(node, self.synchronizer.view().quorum_members().clone());
 
-                            return OPPollResult::QuorumJoined(decisions, joined);
+                            if decisions.is_empty() {
+                                OPExecResult::QuorumJoined(DecisionsAhead::ClearAhead, None, joined)
+                            } else {
+                                OPExecResult::QuorumJoined(DecisionsAhead::ClearAhead, Some(decisions), joined)
+                            }
+                        }
+                        SynchronizerStatus::NewView(consensus_status, decisions) => {
+                            let decisions = self.handle_sync_result(consensus_status, decisions)?;
+
+                            OPExecResult::ProgressedDecision(DecisionsAhead::ClearAhead, decisions)
+                        }
+                        _ => {
+                            warn!("Received sync status that is not handled")
                         }
-                        _ => {}
                     }
                 }
 
@@ -401,15 +404,15 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
                             OPPollResult::RunCst
                         }
                         SyncPhaseRes::SyncProtocolNotNeeded => {
-                            warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, SyncProtocolNotNeeded");
+                            error!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, SyncProtocolNotNeeded");
                             OPPollResult::RePoll
                         }
-                        SyncPhaseRes::JoinedQuorum(_, _) => {
-                            warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, JoinedQuorum");
+                        SyncPhaseRes::JoinedQuorum(_, _, _) => {
+                            error!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, JoinedQuorum");
                             OPPollResult::RePoll
                         }
-                        SyncPhaseRes::SyncProtocolFinished(_) => {
-                            warn!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, Protocol Finished");
+                        SyncPhaseRes::SyncProtocolFinished(_, _) => {
+                            error!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, Protocol Finished");
                             OPPollResult::RePoll
                         }
                     };
@@ -435,13 +438,14 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
             ConsensusPollStatus::Decided(decisions) => {
                 let finalized_decisions = self.finalize_all_possible().expect("Failed to finalize decisions");
 
-                return OPPollResult::Decided(finalized_decisions);
+                let decisions = self.merge_decisions(decisions, finalized_decisions)?;
+
+                OPPollResult::ProgressedDecision(decisions)
             }
         }
     }
 
-    fn update_sync_phase(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>>{
-
+    fn update_sync_phase(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>> {
         match message.message() {
             PBFTMessage::ViewChange(view_change) => {
                 return Ok(match self.adv_sync(message) {
@@ -451,22 +455,23 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
                     SyncPhaseRes::RunSyncProtocol => {
                         OPExecResult::MessageProcessedNoUpdate
                     }
-                    SyncPhaseRes::SyncProtocolFinished(to_execute) => {
-                        match to_execute {
-                            None => {
-                                OPExecResult::MessageProcessedNoUpdate
-                            }
-                            Some(to_execute) => {
-                                OPExecResult::Decided(vec![to_execute])
-                            }
-                        }
+                    SyncPhaseRes::SyncProtocolFinished(status, to_execute) => {
+                        OPExecResult::ProgressedDecision(DecisionsAhead::ClearAhead, self.handle_sync_result(status, to_execute)?)
                     }
-                    SyncPhaseRes::JoinedQuorum(to_execute, node) => {
+                    SyncPhaseRes::JoinedQuorum(status, to_execute, node) => {
                         info!("Replica {:?} joined the quorum, with a decision to execute? {}", node, to_execute.is_some());
 
                         let new_quorum = self.synchronizer.view().quorum_members().clone();
 
-                        OPExecResult::QuorumJoined(to_execute.map(|x| vec![x]), node, new_quorum)
+                        let join_info = JoinInfo::new(node, new_quorum);
+
+                        let decision_adv = self.handle_sync_result(status, to_execute)?;
+
+                        if decision_adv.is_empty() {
+                            OPExecResult::QuorumJoined(DecisionsAhead::ClearAhead, None, join_info)
+                        } else {
+                            OPExecResult::QuorumJoined(DecisionsAhead::ClearAhead, Some(decision_adv), join_info)
+                        }
                     }
                     SyncPhaseRes::RunCSTProtocol => {
                         OPExecResult::RunCst
@@ -482,8 +487,83 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
         Ok(OPExecResult::MessageProcessedNoUpdate)
     }
 
-    fn update_normal_phase(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>>
-        where PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+    fn merge_decisions(&mut self, status: MaybeVec<OPDecision<D::Request>>, finalized_decisions: Vec<ProtocolConsensusDecision<D::Request>>) -> Result<MaybeVec<OPDecision<D::Request>>> {
+        let mut map = BTreeMap::new();
+
+        Self::merge_decision_vec(&mut map, status)?;
+
+        for decision in finalized_decisions {
+            if let Some(member) = map.get_mut(&decision.sequence_number()) {
+                member.append_decision_info(DecisionInfo::DecisionDone(decision));
+            } else {
+                map.insert(decision.sequence_number(), Decision::completed_decision(decision.sequence_number(), decision));
+            }
+        }
+
+        let mut decisions = MaybeVec::builder();
+
+        // By turning this btree map into a vec, we maintain ordering on the delivery (Shouldn't
+        // really be necessary but always nice to have)
+        map.into_iter().for_each(|(seq, decision)| {
+            decisions.push(decision);
+        });
+
+        Ok(decisions.build())
+    }
+
+    /// Handles the result of a synchronizer result
+    fn handle_sync_result(&mut self, status: ConsensusStatus<D::Request>, to_exec: Option<OPDecision<D::Request>>) -> Result<MaybeVec<OPDecision<D::Request>>> {
+        let mut map = BTreeMap::new();
+
+        match status {
+            ConsensusStatus::Deciding(decision) => {
+                Self::merge_decision_vec(&mut map, decision)?;
+            }
+            ConsensusStatus::Decided(decision) => {
+                Self::merge_decision_vec(&mut map, decision)?;
+
+                let finalized = self.finalize_all_possible()?;
+
+                for decision in finalized {
+                    if let Some(member) = map.get_mut(&decision.sequence_number()) {
+                        member.append_decision_info(DecisionInfo::DecisionDone(decision));
+                    } else {
+                        map.insert(decision.sequence_number(), Decision::completed_decision(decision.sequence_number(), decision));
+                    }
+                }
+            }
+            _ => {}
+        }
+
+        if let Some(decision) = to_exec {
+            map.insert(decision.sequence_number(), decision);
+        }
+
+        let mut decisions = MaybeVec::builder();
+
+        // By turning this btree map into a vec, we maintain ordering on the delivery (Shouldn't
+        // really be necessary but always nice to have)
+        map.into_iter().for_each(|(seq, decision)| {
+            decisions.push(decision);
+        });
+
+        Ok(decisions.build())
+    }
+
+    /// Merge a decision vector with the already existing btreemap
+    fn merge_decision_vec(map: &mut BTreeMap<SeqNo, OPDecision<<D as ApplicationData>::Request>>, decision: MaybeVec<OPDecision<<D as ApplicationData>::Request>>) -> Result<()> {
+        for dec in decision.into_iter() {
+            if let Some(member) = map.get_mut(&dec.sequence_number()) {
+                member.merge_decisions(dec)?;
+            } else {
+                map.insert(dec.sequence_number(), dec);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn update_normal_phase(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>> {
         match message.message() {
             PBFTMessage::Consensus(_) => {
                 return self.adv_consensus(message);
@@ -545,18 +625,14 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
                 OPExecResult::MessageQueued
             }
             ConsensusStatus::Deciding(result) => {
-                OPExecResult::ProgressedDecision(result)
+                OPExecResult::ProgressedDecision(DecisionsAhead::Ignore, result)
             }
             ConsensusStatus::Decided(result) => {
                 let finalized_decisions = self.finalize_all_possible()?;
 
-                let mut decision_builder = MaybeVec::builder();
+                let decision = self.merge_decisions(result, finalized_decisions)?;
 
-                for decision in finalized_decisions.into_iter() {
-                    decision_builder.push(Decision::completed_decision(decision.sequence_number(), decision));
-                }
-
-                OPExecResult::Decided(decision_builder.build())
+                OPExecResult::ProgressedDecision(DecisionsAhead::Ignore, decision)
             }
         });
     }
@@ -571,10 +647,8 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
             // This will automatically move the consensus machine to the next consensus instance
             let completed_batch = self.consensus.finalize(&view)?.unwrap();
 
-            let seq = completed_batch.sequence_number();
-
             //Should the execution be scheduled here or will it be scheduled by the persistent log?
-            let exec_info = self.message_log.finalize_batch(seq, completed_batch)?;
+            let exec_info = self.message_log.finalize_batch(completed_batch)?;
 
             finalized_decisions.push(exec_info);
         }
@@ -599,14 +673,21 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
         return match status {
             SynchronizerStatus::Nil => SyncPhaseRes::SyncProtocolNotNeeded,
             SynchronizerStatus::Running => SyncPhaseRes::RunSyncProtocol,
-            SynchronizerStatus::NewView(to_execute) => {
+            SynchronizerStatus::NewView(consensus_status, to_execute) => {
                 //Our current view has been updated and we have no more state operations
                 //to perform. This happens if we are a correct replica and therefore do not need
                 //To update our state or if we are a replica that was incorrect and whose state has
                 //Already been updated from the Cst protocol
                 self.switch_phase(ConsensusPhase::NormalPhase);
 
-                SyncPhaseRes::SyncProtocolFinished(to_execute)
+                SyncPhaseRes::SyncProtocolFinished(consensus_status, to_execute)
+            }
+            SynchronizerStatus::NewViewJoinedQuorum(consensus_decision, decision, node) => {
+                //We have joined a quorum and we have a new view to execute
+                //We need to switch to the normal phase and execute the new view
+                self.switch_phase(ConsensusPhase::NormalPhase);
+
+                SyncPhaseRes::JoinedQuorum(consensus_decision, decision, node)
             }
             SynchronizerStatus::RunCst => {
                 //This happens when a new view is being introduced and we are not up to date
@@ -621,13 +702,6 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
 
                 SyncPhaseRes::RunCSTProtocol
             }
-            SynchronizerStatus::NewViewJoinedQuorum(decision, node) => {
-                //We have joined a quorum and we have a new view to execute
-                //We need to switch to the normal phase and execute the new view
-                self.switch_phase(ConsensusPhase::NormalPhase);
-
-                SyncPhaseRes::JoinedQuorum(decision, node)
-            }
             // should not happen...
             _ => {
                 unreachable!()
@@ -636,70 +710,6 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
     }
 }
 
-impl<D, NT, > StatefulOrderProtocol<D, NT, PL> for PBFTOrderProtocol<D, NT>
-    where D: ApplicationData + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
-    type StateSerialization = PBFTConsensus<D>;
-
-    fn initialize_with_initial_state(config: Self::Config,
-                                     args: OrderingProtocolArgs<D, NT>,
-                                     initial_state: DecisionLog<D::Request>) -> Result<Self> where Self: Sized {
-        Self::initialize_protocol(config, args, Some(initial_state))
-    }
-
-    fn install_state(&mut self,
-                     view_info: View<Self::PermissionedSerialization>,
-                     dec_log: DecLog<D, Self::Serialization, Self::StateSerialization>) -> Result<Vec<D::Request>>
-        where PL: StatefulOrderingProtocolLog<D, PBFTConsensus<D>, PBFTConsensus<D>, PBFTConsensus<D>> {
-        info!("{:?} // Installing decision log with Seq No {:?} and View {:?}", self.node.id(),
-                dec_log.sequence_number(), view_info);
-
-        let last_exec = if let Some(last_exec) = dec_log.last_execution() {
-            last_exec
-        } else {
-            SeqNo::ZERO
-        };
-
-        info!("{:?} // Installing decision log with last execution {:?}", self.node.id(),last_exec);
-
-        if self.synchronizer.received_view_from_state_transfer(view_info.clone()) {
-            info!("{:?} // We have pending view state messages that we need to process. Switching to sync phase", self.node.id());
-            self.switch_phase(ConsensusPhase::SyncPhase);
-        }
-
-        let start = Instant::now();
-
-        let res = self.consensus.install_state(view_info.clone(), &dec_log)?;
-
-        metric_duration(CONSENSUS_INSTALL_STATE_TIME_ID, start.elapsed());
-
-        let start = Instant::now();
-
-        self.message_log.install_state(view_info, dec_log);
-
-        metric_duration(MSG_LOG_INSTALL_TIME_ID, start.elapsed());
-
-        Ok(res)
-    }
-
-    fn snapshot_log(&mut self) -> Result<(View<Self::PermissionedSerialization>, DecLog<D, Self::Serialization, Self::StateSerialization>)> {
-        self.message_log.snapshot(self.synchronizer.view())
-    }
-
-    fn current_log(&self) -> Result<&DecLog<D, Self::Serialization, Self::StateSerialization>>
-        where PL: StatefulOrderingProtocolLog<D, Self::Serialization, Self::StateSerialization, Self::PermissionedSerialization> {
-        Ok(self.message_log.decision_log())
-    }
-
-    fn checkpointed(&mut self, seq_no: SeqNo) -> Result<()> {
-        self.message_log.finalize_checkpoint(seq_no)
-    }
-
-    fn get_proof(&self, seq: SeqNo) -> Result<Option<SerProof<D, Self::Serialization>>> {
-        todo!()
-    }
-}
-
 impl<D, NT> PBFTOrderProtocol<D, NT>
     where D: ApplicationData + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>> + 'static, {
@@ -859,8 +869,8 @@ impl<D, NT> LoggableOrderProtocol<D, NT> for PBFTOrderProtocol<D, NT>
         (proof.metadata(), messages)
     }
 
-    fn get_requests_in_proof(proof: &PProof<D, Self::Serialization, Self::PersistableTypes>) -> (UpdateBatch<D::Request>, Vec<ClientRqInfo>) {
-        todo!()
+    fn get_requests_in_proof(proof: &PProof<D, Self::Serialization, Self::PersistableTypes>) -> Result<ProtocolConsensusDecision<D::Request>> {
+        Ok(ProtocolConsensusDecision::from(proof))
     }
 }
 
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index ab8ba438..abe716bd 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -37,7 +37,7 @@ use crate::bft::log::decisions::{CollectData, Proof, ProofMetadata, ViewDecision
 use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, FwdConsensusMessage, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
-use crate::bft::{PBFT, PBFTOrderProtocol};
+use crate::bft::{OPDecision, PBFT, PBFTOrderProtocol};
 use crate::bft::sync::view::ViewInfo;
 
 use self::{follower_sync::FollowerSynchronizer, replica_sync::ReplicaSynchronizer};
@@ -368,11 +368,10 @@ pub enum SynchronizerStatus<O> {
     /// The view change protocol is currently running.
     Running,
     /// The view change protocol just finished running.
-    NewView(ConsensusStatus<O>, Option<Decision<ProofMetadata, PBFTMessage<O>, O>>),
+    NewView(ConsensusStatus<O>, Option<OPDecision<O>>),
     /// The view change protocol just finished running and we
     /// have successfully joined the quorum.
-    NewViewJoinedQuorum(ConsensusStatus<O>, Option<Decision<ProofMetadata, PBFTMessage<O>, O>>, NodeId),
-
+    NewViewJoinedQuorum(ConsensusStatus<O>, Option<OPDecision<O>>, NodeId),
     /// Before we finish the view change protocol, we need
     /// to run the CST protocol.
     RunCst,
@@ -1639,14 +1638,10 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
             // We are missing the last decision, which should be included in the collect data
             // sent by the leader in the SYNC message
             let to_execute = if let Some(last_proof) = last_proof {
-                let proof = last_proof.metadata().clone();
-                let messages = last_proof.all_messages();
-                let quorum_result = consensus.catch_up_to_quorum(last_proof.seq_no(), &view, last_proof, log)
+                let quorum_result = consensus.catch_up_to_quorum(&view, last_proof, log)
                     .expect("Failed to catch up to quorum");
 
-                let decision = Decision::full_decision_info(last_executed_cid, proof, messages, quorum_result);
-
-                Some(decision)
+                Some(quorum_result)
             } else {
                 // This maybe happens when a checkpoint is done and the first execution after it
                 // fails, leading to a view change? Don't really know how this would be possible

From 2796b23203a280136b1636d50f73449ebb86c0cd Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Fri, 13 Oct 2023 19:36:13 +0100
Subject: [PATCH 60/80] Started to fix compilation issues. Fixed common.

---
 .../src/bft/log/decided/mod.rs                |  96 ++++++++++++++-
 .../src/bft/log/decisions/mod.rs              | 115 ------------------
 2 files changed, 92 insertions(+), 119 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/log/decided/mod.rs b/febft-pbft-consensus/src/bft/log/decided/mod.rs
index 019461a5..cc1b5443 100644
--- a/febft-pbft-consensus/src/bft/log/decided/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decided/mod.rs
@@ -1,16 +1,17 @@
+use atlas_common::ordering::{Orderable, SeqNo};
 use crate::bft::log::decisions::Proof;
 
 /// A necessary decision log for the ability to perform view changes.
 /// Only stores the latest performed decision
-pub struct OngoingDecisionLog<O> {
+pub struct DecisionLog<O> {
     /// The last decision that was performed by the ordering protocol
     last_decision: Option<Proof<O>>
 }
 
-impl<O> OngoingDecisionLog<O> {
+impl<O> DecisionLog<O> {
 
     fn init(last_proof: Option<Proof<O>>) -> Self {
-        OngoingDecisionLog {
+        DecisionLog {
             last_decision: last_proof,
         }
     }
@@ -25,4 +26,91 @@ impl<O> OngoingDecisionLog<O> {
         self.last_decision.clone()
     }
     
-}
\ No newline at end of file
+}
+
+impl<O> DecisionLog<O> {
+    pub fn new() -> Self {
+        Self {
+            last_decision: None,
+        }
+    }
+
+    pub fn from_decided(proof: Proof<O>) -> Self {
+        Self {
+            last_decision: Some(proof),
+        }
+    }
+
+    pub fn from_proofs(mut proofs: Vec<Proof<O>>) -> Self {
+
+        proofs.sort_by(|a, b| a.sequence_number().cmp(&b.sequence_number()).reverse());
+
+        let last_decided = proofs.first().map(|proof| proof.sequence_number());
+
+        Self {
+            last_exec: last_decided,
+            decided: proofs,
+        }
+    }
+
+    /// Returns the sequence number of the last executed batch of client
+    /// requests, assigned by the conesensus layer.
+    pub fn last_execution(&self) -> Option<SeqNo> {
+        self.last_decision.map(|proof| proof.sequence_number())
+    }
+
+    /// Append a proof to the end of the log. Assumes all prior checks have been done
+    pub(crate) fn append_proof(&mut self, proof: Proof<O>) {
+        self.last_decision = Some(proof);
+    }
+
+    //TODO: Maybe make these data structures a BTreeSet so that the messages are always ordered
+    //By their seq no? That way we cannot go wrong in the ordering of messages.
+    pub(crate) fn finished_quorum_execution(&mut self, completed_batch: &CompletedBatch<O>, seq_no: SeqNo, f: usize) -> Result<()> {
+        self.last_exec.replace(seq_no);
+
+        let proof = completed_batch.proof(Some(f))?;
+
+        self.decided.push(proof);
+
+        Ok(())
+    }
+    /// Returns the proof of the last executed consensus
+    /// instance registered in this `DecisionLog`.
+    pub fn last_decision(&self) -> Option<Proof<O>> {
+        self.decided.last().map(|p| (*p).clone())
+    }
+
+    /// Clear the decision log until the given sequence number
+    pub(crate) fn clear_until_seq(&mut self, seq_no: SeqNo) -> usize {
+        let mut net_decided = Vec::with_capacity(self.decided.len());
+
+        let mut decided_request_count = 0;
+
+        let prev_decided = std::mem::replace(&mut self.decided, net_decided);
+
+        for proof in prev_decided.into_iter().rev() {
+            if proof.seq_no <= seq_no {
+                for pre_prepare in &proof.pre_prepares {
+                    //Mark the requests contained in this message for removal
+                    decided_request_count += match pre_prepare.message().kind() {
+                        ConsensusMessageKind::PrePrepare(messages) => messages.len(),
+                        _ => 0,
+                    };
+                }
+            } else {
+                self.decided.push(proof);
+            }
+        }
+
+        self.decided.reverse();
+
+        decided_request_count
+    }
+}
+
+impl<O> Orderable for DecisionLog<O> {
+    fn sequence_number(&self) -> SeqNo {
+        self.last_exec.unwrap_or(SeqNo::ZERO)
+    }
+}
diff --git a/febft-pbft-consensus/src/bft/log/decisions/mod.rs b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
index 282308bf..0d7f6f53 100644
--- a/febft-pbft-consensus/src/bft/log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
@@ -26,19 +26,6 @@ pub struct Decision<O> {
     commits: Vec<StoredConsensusMessage<O>>,
 }
 
-/// Contains all the decisions the consensus has decided since the last checkpoint.
-/// The currently deciding variable contains the decision that is currently ongoing.
-///
-/// Cloning this decision log is actually pretty cheap (compared to the alternative of cloning
-/// all requests executed since the last checkpoint) since it only has to clone the arcs (which is one atomic operation)
-/// We can't wrap the entire vector since the decision log is actually constantly modified by the consensus
-#[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
-#[derive(Clone)]
-pub struct DecisionLog<O> {
-    last_exec: Option<SeqNo>,
-    decided: Vec<Proof<O>>,
-}
-
 /// Metadata about a proof
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
 #[derive(Clone, Debug)]
@@ -302,108 +289,6 @@ impl<O> CollectData<O> {
     }
 }
 
-impl<O> DecisionLog<O> {
-    pub fn new() -> Self {
-        Self {
-            last_exec: None,
-            decided: vec![],
-        }
-    }
-
-    pub fn from_decided(last_exec: SeqNo, proofs: Vec<Proof<O>>) -> Self {
-        Self {
-            last_exec: Some(last_exec),
-            decided: proofs,
-        }
-    }
-    
-    pub fn from_proofs(mut proofs: Vec<Proof<O>>) -> Self { 
-        
-        proofs.sort_by(|a, b| a.sequence_number().cmp(&b.sequence_number()).reverse());
-        
-        let last_decided = proofs.first().map(|proof| proof.sequence_number());
-        
-        Self {
-            last_exec: last_decided,
-            decided: proofs,
-        }
-    }
-
-    /// Returns the sequence number of the last executed batch of client
-    /// requests, assigned by the conesensus layer.
-    pub fn last_execution(&self) -> Option<SeqNo> {
-        self.last_exec
-    }
-
-    /// Get all of the decided proofs in this decisionn log
-    pub fn proofs(&self) -> &[Proof<O>] {
-        &self.decided[..]
-    }
-
-    /// Append a proof to the end of the log. Assumes all prior checks have been done
-    pub(crate) fn append_proof(&mut self, proof: Proof<O>) {
-        self.last_exec = Some(proof.seq_no());
-
-        self.decided.push(proof);
-    }
-
-    //TODO: Maybe make these data structures a BTreeSet so that the messages are always ordered
-    //By their seq no? That way we cannot go wrong in the ordering of messages.
-    pub(crate) fn finished_quorum_execution(&mut self, completed_batch: &CompletedBatch<O>, seq_no: SeqNo, f: usize) -> Result<()> {
-        self.last_exec.replace(seq_no);
-
-        let proof = completed_batch.proof(Some(f))?;
-
-        self.decided.push(proof);
-
-        Ok(())
-    }
-    /// Returns the proof of the last executed consensus
-    /// instance registered in this `DecisionLog`.
-    pub fn last_decision(&self) -> Option<Proof<O>> {
-        self.decided.last().map(|p| (*p).clone())
-    }
-
-    /// Clear the decision log until the given sequence number
-    pub(crate) fn clear_until_seq(&mut self, seq_no: SeqNo) -> usize {
-        let mut net_decided = Vec::with_capacity(self.decided.len());
-
-        let mut decided_request_count = 0;
-
-        let prev_decided = std::mem::replace(&mut self.decided, net_decided);
-
-        for proof in prev_decided.into_iter().rev() {
-            if proof.seq_no <= seq_no {
-                for pre_prepare in &proof.pre_prepares {
-                    //Mark the requests contained in this message for removal
-                    decided_request_count += match pre_prepare.message().kind() {
-                        ConsensusMessageKind::PrePrepare(messages) => messages.len(),
-                        _ => 0,
-                    };
-                }
-            } else {
-                self.decided.push(proof);
-            }
-        }
-
-        self.decided.reverse();
-
-        decided_request_count
-    }
-}
-
-impl<O> Orderable for DecisionLog<O> {
-    fn sequence_number(&self) -> SeqNo {
-        self.last_exec.unwrap_or(SeqNo::ZERO)
-    }
-}
-
-impl<O> OrderProtocolLog for DecisionLog<O> {
-    fn first_seq(&self) -> Option<SeqNo> {
-        self.proofs().first().map(|p| p.sequence_number())
-    }
-}
-
 impl<O> OrderProtocolProof for Proof<O> {
     fn contained_messages(&self) -> usize {
         self.metadata().contained_client_rqs()

From e5e5d8faae2558f3a7bd8937ff4d6f9259f7183e Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Sat, 14 Oct 2023 17:55:07 +0100
Subject: [PATCH 61/80] More compilation issues fixed, having some issues with
 the ordering inheritance of shareable messages.

---
 febft-pbft-consensus/src/bft/config/mod.rs    |  2 +-
 .../src/bft/consensus/decision/mod.rs         | 31 +++----
 febft-pbft-consensus/src/bft/consensus/mod.rs | 20 ++---
 .../src/bft/log/decided/mod.rs                | 90 +++----------------
 .../src/bft/log/deciding/mod.rs               | 18 ++--
 .../src/bft/log/decisions/mod.rs              |  8 +-
 febft-pbft-consensus/src/bft/log/mod.rs       | 15 +++-
 febft-pbft-consensus/src/bft/message/mod.rs   |  2 +-
 .../src/bft/message/serialize/mod.rs          | 87 +++++++-----------
 febft-pbft-consensus/src/bft/mod.rs           | 81 +++++++++--------
 .../src/bft/sync/follower_sync/mod.rs         |  8 +-
 febft-pbft-consensus/src/bft/sync/mod.rs      | 28 +++---
 .../src/bft/sync/replica_sync/mod.rs          | 21 ++---
 febft-pbft-consensus/src/bft/sync/view/mod.rs |  2 +-
 febft-state-transfer/src/lib.rs               |  6 +-
 .../src/message/serialize/capnp/mod.rs        |  4 +-
 .../src/message/serialize/mod.rs              |  2 +-
 17 files changed, 170 insertions(+), 255 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/config/mod.rs b/febft-pbft-consensus/src/bft/config/mod.rs
index 1564b707..40531307 100644
--- a/febft-pbft-consensus/src/bft/config/mod.rs
+++ b/febft-pbft-consensus/src/bft/config/mod.rs
@@ -2,7 +2,7 @@ use std::time::Duration;
 
 use atlas_common::node_id::NodeId;
 use atlas_core::followers::FollowerHandle;
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::sync::view::ViewInfo;
diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index 5a363e64..ddc21502 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -7,25 +7,21 @@ use chrono::Utc;
 use log::{debug, info, warn};
 
 use atlas_common::error::*;
-use atlas_common::maybe_vec::MaybeVec;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_communication::message::{Header, StoredMessage};
+use atlas_communication::message::Header;
 use atlas_core::messages::ClientRqInfo;
-use atlas_core::ordering_protocol::{Decision, DecisionInfo};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_core::persistent_log::{OperationMode, OrderingProtocolLog};
 use atlas_core::smr::smr_decision_log::ShareableMessage;
 use atlas_core::timeouts::Timeouts;
-use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::metric_duration;
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::consensus::accessory::{AccessoryConsensus, ConsensusDecisionAccessory};
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
 use crate::bft::log::deciding::{CompletedBatch, WorkingDecisionLog};
-use crate::bft::log::decisions::{IncompleteProof, ProofMetadata};
+use crate::bft::log::decisions::IncompleteProof;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
-use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{ConsensusMetrics, PRE_PREPARE_ANALYSIS_ID};
 use crate::bft::PBFT;
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer};
@@ -37,9 +33,8 @@ macro_rules! extract_msg {
     };
     ($rsp:expr, $g:expr, $q:expr) => {
         if let Some(stored) = $q.pop_front() {
-            let (header, message) = stored.into_inner();
 
-            DecisionPollStatus::NextMessage(header, message)
+            DecisionPollStatus::NextMessage(stored)
         } else {
             *$g = false;
             $rsp
@@ -202,7 +197,7 @@ impl<D> ConsensusDecision<D>
     }
 
     pub fn queue(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) {
-        match message.kind() {
+        match message.message().consensus().kind() {
             ConsensusMessageKind::PrePrepare(_) => {
                 self.message_queue.queue_pre_prepare(message);
             }
@@ -333,6 +328,7 @@ impl<D> ConsensusDecision<D>
 
                 //TODO: Try out cloning each request on this method,
                 let mut digests = request_batch_received(
+                    header,
                     message,
                     timeouts,
                     synchronizer,
@@ -348,7 +344,7 @@ impl<D> ConsensusDecision<D>
                     let batch_metadata = batch_metadata.unwrap();
 
                     info!("{:?} // Completed pre prepare phase with all pre prepares Seq {:?} with pre prepare from {:?}. Batch size {:?}",
-                        node.id(), self.sequence_number(), header.from(), self.message_log.current_batch_size());
+                        node.id(), self.sequence_number(), header.from(), self.working_log.current_batch_size());
 
                     //We have received all pre prepare requests for this consensus instance
                     //We are now ready to broadcast our prepare message and move to the next phase
@@ -377,7 +373,7 @@ impl<D> ConsensusDecision<D>
                     DecisionPhase::Preparing(0)
                 } else {
                     debug!("{:?} // Received pre prepare message {:?} from {:?}. Current received {:?}",
-                        self.node_id, stored_msg.message(), stored_msg.header().from(), received);
+                        self.node_id, s_message.message(), s_message.header().from(), received);
 
                     self.accessory.handle_partial_pre_prepare(&self.working_log, &view,
                                                               &header, &message, &**node);
@@ -460,7 +456,7 @@ impl<D> ConsensusDecision<D>
                     DecisionPhase::Committing(0)
                 } else {
                     debug!("{:?} // Received prepare message {:?} from {:?}. Current count {}",
-                        self.node_id, stored_msg.message().sequence_number(), header.from(), received);
+                        self.node_id, s_message.message().sequence_number(), header.from(), received);
 
                     self.accessory.handle_preparing_no_quorum(&self.working_log, &view,
                                                               &header, &message, &**node);
@@ -491,7 +487,7 @@ impl<D> ConsensusDecision<D>
                     ConsensusMessageKind::Commit(d) if *d != self.working_log.current_digest().unwrap() => {
                         // drop msg with different digest from proposed value
                         warn!("{:?} // Dropped commit message {:?} from {:?} because of digest {:?} vs {:?} (ours)",
-                            self.node_id, message.sequence_number(), header.from(), d, self.message_log.current_digest());
+                            self.node_id, message.sequence_number(), header.from(), d, self.working_log.current_digest());
 
                         return Ok(DecisionStatus::MessageIgnored);
                     }
@@ -527,7 +523,7 @@ impl<D> ConsensusDecision<D>
                     Ok(DecisionStatus::Decided(s_message))
                 } else {
                     debug!("{:?} // Received commit message {:?} from {:?}. Current count {}",
-                        self.node_id, stored_msg.message().sequence_number(), header.from(), received);
+                        self.node_id, s_message.message().sequence_number(), header.from(), received);
 
                     self.phase = DecisionPhase::Committing(received);
 
@@ -581,6 +577,7 @@ impl<D> Orderable for ConsensusDecision<D>
 
 #[inline]
 fn request_batch_received<D>(
+    header: &Header,
     pre_prepare: &ConsensusMessage<D::Request>,
     timeouts: &Timeouts,
     synchronizer: &Synchronizer<D>,
@@ -593,7 +590,7 @@ fn request_batch_received<D>(
 
     let mut batch_guard = log.batch_meta().lock().unwrap();
 
-    batch_guard.batch_size += match pre_prepare.message().kind() {
+    batch_guard.batch_size += match pre_prepare.kind() {
         ConsensusMessageKind::PrePrepare(req) => {
             req.len()
         }
@@ -603,7 +600,7 @@ fn request_batch_received<D>(
     batch_guard.reception_time = Utc::now();
 
     // Notify the synchronizer that a batch has been received
-    let digests = synchronizer.request_batch_received(pre_prepare, timeouts);
+    let digests = synchronizer.request_batch_received(header, pre_prepare, timeouts);
 
     metric_duration(PRE_PREPARE_ANALYSIS_ID, start.elapsed());
 
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 1c5a81b2..aea07cb9 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -13,25 +13,24 @@ use atlas_common::maybe_vec::MaybeVec;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_advance_message_queue_return, tbo_queue_message};
 use atlas_communication::message::{Header, StoredMessage};
-use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage};
+use atlas_core::ordering_protocol::Decision;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_core::ordering_protocol::{Decision, ProtocolConsensusDecision};
-use atlas_core::persistent_log::{OrderingProtocolLog};
 use atlas_core::smr::smr_decision_log::ShareableMessage;
 use atlas_core::timeouts::Timeouts;
+use atlas_metrics::metrics::metric_increment;
 use atlas_smr_application::ExecutorHandle;
 use atlas_smr_application::serialize::ApplicationData;
-use atlas_metrics::metrics::metric_increment;
 
 use crate::bft::{OPDecision, PBFT, SysMsg};
 use crate::bft::consensus::decision::{ConsensusDecision, DecisionPollStatus, DecisionStatus, MessageQueue};
+use crate::bft::log::decided::DecisionLog;
 use crate::bft::log::deciding::CompletedBatch;
-use crate::bft::log::decisions::{DecisionLog, IncompleteProof, Proof, ProofMetadata};
+use crate::bft::log::decisions::{IncompleteProof, Proof, ProofMetadata};
 use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::metric::OPERATIONS_PROCESSED_ID;
-use crate::bft::sync::{AbstractSynchronizer, Synchronizer};
+use crate::bft::sync::{Synchronizer};
 use crate::bft::sync::view::ViewInfo;
 
 pub mod decision;
@@ -437,12 +436,11 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
                 ConsensusStatus::Deciding(MaybeVec::from_one(Decision::decision_info_from_message(decision_seq, message)))
             }
             DecisionStatus::Decided(message) => {
-                ConsensusStatus::Decided
-            }
-            DecisionStatus::DecidedIgnored => {}
-            DecisionStatus::MessageIgnored => {
-                ConsensusStatus::MessageIgnored
+                ConsensusStatus::Decided(MaybeVec::from_one(Decision::decision_info_from_message(decision_seq, message)))
             }
+            DecisionStatus::DecidedIgnored => ConsensusStatus::Decided(MaybeVec::None),
+            DecisionStatus::MessageIgnored => ConsensusStatus::MessageIgnored
+
         })
     }
 
diff --git a/febft-pbft-consensus/src/bft/log/decided/mod.rs b/febft-pbft-consensus/src/bft/log/decided/mod.rs
index cc1b5443..e10092a3 100644
--- a/febft-pbft-consensus/src/bft/log/decided/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decided/mod.rs
@@ -1,4 +1,6 @@
 use atlas_common::ordering::{Orderable, SeqNo};
+use atlas_common::error::*;
+use crate::bft::log::deciding::CompletedBatch;
 use crate::bft::log::decisions::Proof;
 
 /// A necessary decision log for the ability to perform view changes.
@@ -10,107 +12,35 @@ pub struct DecisionLog<O> {
 
 impl<O> DecisionLog<O> {
 
-    fn init(last_proof: Option<Proof<O>>) -> Self {
+    pub(crate) fn init(last_proof: Option<Proof<O>>) -> Self {
         DecisionLog {
             last_decision: last_proof,
         }
     }
 
     /// Install a given proof
-    fn install_proof(&mut self, proof: Proof<O>) {
+    pub fn install_proof(&mut self, proof: Proof<O>) {
         self.last_decision = Some(proof)
     }
 
     /// Get the last decision
-    fn last_decision(&self) -> Option<Proof<O>>  {
+    pub fn last_decision(&self) -> Option<Proof<O>>  {
         self.last_decision.clone()
     }
-    
-}
-
-impl<O> DecisionLog<O> {
-    pub fn new() -> Self {
-        Self {
-            last_decision: None,
-        }
-    }
-
-    pub fn from_decided(proof: Proof<O>) -> Self {
-        Self {
-            last_decision: Some(proof),
-        }
-    }
 
-    pub fn from_proofs(mut proofs: Vec<Proof<O>>) -> Self {
-
-        proofs.sort_by(|a, b| a.sequence_number().cmp(&b.sequence_number()).reverse());
-
-        let last_decided = proofs.first().map(|proof| proof.sequence_number());
-
-        Self {
-            last_exec: last_decided,
-            decided: proofs,
-        }
-    }
-
-    /// Returns the sequence number of the last executed batch of client
-    /// requests, assigned by the conesensus layer.
     pub fn last_execution(&self) -> Option<SeqNo> {
-        self.last_decision.map(|proof| proof.sequence_number())
+        self.last_decision.map(|decision| decision.sequence_number())
     }
 
-    /// Append a proof to the end of the log. Assumes all prior checks have been done
-    pub(crate) fn append_proof(&mut self, proof: Proof<O>) {
-        self.last_decision = Some(proof);
-    }
-
-    //TODO: Maybe make these data structures a BTreeSet so that the messages are always ordered
-    //By their seq no? That way we cannot go wrong in the ordering of messages.
-    pub(crate) fn finished_quorum_execution(&mut self, completed_batch: &CompletedBatch<O>, seq_no: SeqNo, f: usize) -> Result<()> {
-        self.last_exec.replace(seq_no);
-
-        let proof = completed_batch.proof(Some(f))?;
-
-        self.decided.push(proof);
-
-        Ok(())
-    }
-    /// Returns the proof of the last executed consensus
-    /// instance registered in this `DecisionLog`.
-    pub fn last_decision(&self) -> Option<Proof<O>> {
-        self.decided.last().map(|p| (*p).clone())
+    pub fn append_proof(&mut self, proof: Proof<O>) {
+        self.last_decision = Some(proof)
     }
 
-    /// Clear the decision log until the given sequence number
-    pub(crate) fn clear_until_seq(&mut self, seq_no: SeqNo) -> usize {
-        let mut net_decided = Vec::with_capacity(self.decided.len());
-
-        let mut decided_request_count = 0;
-
-        let prev_decided = std::mem::replace(&mut self.decided, net_decided);
-
-        for proof in prev_decided.into_iter().rev() {
-            if proof.seq_no <= seq_no {
-                for pre_prepare in &proof.pre_prepares {
-                    //Mark the requests contained in this message for removal
-                    decided_request_count += match pre_prepare.message().kind() {
-                        ConsensusMessageKind::PrePrepare(messages) => messages.len(),
-                        _ => 0,
-                    };
-                }
-            } else {
-                self.decided.push(proof);
-            }
-        }
-
-        self.decided.reverse();
-
-        decided_request_count
-    }
 }
 
+
 impl<O> Orderable for DecisionLog<O> {
     fn sequence_number(&self) -> SeqNo {
-        self.last_exec.unwrap_or(SeqNo::ZERO)
+        self.last_decision.map(|f| f.sequence_number()).unwrap_or(SeqNo::ZERO)
     }
 }
diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
index 8d91ebdb..4e49f14e 100644
--- a/febft-pbft-consensus/src/bft/log/deciding/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -128,7 +128,7 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
     pub fn new(node: NodeId, seq: SeqNo, view: &ViewInfo) -> Self {
         let leader_count = view.leader_set().len();
         Self {
-            node_id,
+            node_id: node,
             seq_no: seq,
             duplicate_detection: Default::default(),
             batch_digest: None,
@@ -217,13 +217,13 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
     }
 
     /// Process the message received
-    pub(crate) fn process_message<O>(&mut self, header: &Header, message: &ConsensusMessage<O>) -> Result<()> {
-        match message.message().kind() {
+    pub(crate) fn process_message(&mut self, header: &Header, message: &ConsensusMessage<O>) -> Result<()> {
+        match message.kind() {
             ConsensusMessageKind::Prepare(_) => {
-                self.duplicate_detection.insert_prepare_received(message.header().from())?;
+                self.duplicate_detection.insert_prepare_received(header.from())?;
             }
             ConsensusMessageKind::Commit(_) => {
-                self.duplicate_detection.insert_commit_received(message.header().from())?;
+                self.duplicate_detection.insert_commit_received(header.from())?;
             }
             _ => unreachable!()
         }
@@ -303,6 +303,14 @@ impl<O> Orderable for CompletedBatch<O> {
     }
 }
 
+impl<O> CompletedBatch<O> {
+
+    pub fn request_count(&self) -> usize {
+        self.client_requests.len()
+    }
+
+}
+
 impl<O> Orderable for WorkingDecisionLog<O> {
     fn sequence_number(&self) -> SeqNo {
         self.seq_no
diff --git a/febft-pbft-consensus/src/bft/log/decisions/mod.rs b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
index 0d7f6f53..5d0a62c9 100644
--- a/febft-pbft-consensus/src/bft/log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
@@ -1,6 +1,5 @@
 use std::fmt::{Debug, Formatter};
 use std::ops::Deref;
-use std::sync::Arc;
 
 #[cfg(feature = "serialize_serde")]
 use serde::{Deserialize, Serialize};
@@ -8,12 +7,11 @@ use serde::{Deserialize, Serialize};
 use atlas_common::crypto::hash::Digest;
 use atlas_common::error::*;
 use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_communication::message::StoredMessage;
-use atlas_core::ordering_protocol::networking::serialize::{OrderProtocolLog, OrderProtocolProof};
+use atlas_core::ordering_protocol::networking::serialize::OrderProtocolProof;
 use atlas_core::smr::smr_decision_log::ShareableMessage;
-use crate::bft::log::deciding::CompletedBatch;
 
-use crate::bft::message::{ConsensusMessageKind, PBFTMessage};
+use crate::bft::message::PBFTMessage;
+
 pub type StoredConsensusMessage<O> = ShareableMessage<PBFTMessage<O>>;
 
 #[cfg_attr(feature = "serialize_serde", derive(Serialize, Deserialize))]
diff --git a/febft-pbft-consensus/src/bft/log/mod.rs b/febft-pbft-consensus/src/bft/log/mod.rs
index cda43ea9..c87b59f6 100644
--- a/febft-pbft-consensus/src/bft/log/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/mod.rs
@@ -1,14 +1,17 @@
 use either::Either;
-use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{InvalidSeqNo, Orderable, SeqNo};
+
 use atlas_common::error::*;
+use atlas_common::node_id::NodeId;
+use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_communication::message::Header;
 use atlas_core::messages::{ClientRqInfo, RequestMessage};
 use atlas_core::ordering_protocol::{Decision, ProtocolConsensusDecision};
 use atlas_smr_application::app::UpdateBatch;
 use atlas_smr_application::serialize::ApplicationData;
+
+use crate::bft::log::decided::DecisionLog;
 use crate::bft::log::deciding::{CompletedBatch, FinishedMessageLog};
-use crate::bft::log::decisions::{DecisionLog, Proof, ProofMetadata};
+use crate::bft::log::decisions::{Proof, ProofMetadata};
 use crate::bft::message::ConsensusMessageKind;
 use crate::bft::OPDecision;
 
@@ -25,6 +28,10 @@ impl<D> Log<D> where D: ApplicationData {
         &self.decided
     }
 
+    pub fn last_proof(&self) -> Option<Proof<D::Request>> {
+        self.decided.last_decision()
+    }
+
     pub fn install_proof(&mut self, proof: Proof<D::Request>) -> Result<OPDecision<D::Request>> {
         if let Some(decision) = self.decision_log().last_execution() {
             match proof.seq_no().index(decision) {
@@ -91,7 +98,7 @@ impl<D> Log<D> where D: ApplicationData {
 
 pub fn initialize_decided_log<D>(node_id: NodeId) -> Log<D> where D: ApplicationData {
     Log {
-        decided: DecisionLog::new(),
+        decided: DecisionLog::init(None),
     }
 }
 
diff --git a/febft-pbft-consensus/src/bft/message/mod.rs b/febft-pbft-consensus/src/bft/message/mod.rs
index f0b12d80..61ef3ef3 100644
--- a/febft-pbft-consensus/src/bft/message/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/mod.rs
@@ -19,7 +19,7 @@ use atlas_communication::message::Header;
 use atlas_core::messages::{RequestMessage, StoredRequestMessage};
 use atlas_smr_application::serialize::ApplicationData;
 
-use crate::bft::msg_log::decisions::CollectData;
+use crate::bft::log::decisions::CollectData;
 use crate::bft::sync::LeaderCollects;
 use crate::bft::sync::view::ViewInfo;
 
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index f1f48c50..a8a7d506 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -20,14 +20,13 @@ use atlas_communication::message::Header;
 use atlas_communication::message_signing::NetworkMessageSignatureVerifier;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_communication::serialize::Serializable;
-use atlas_core::messages::SystemMessage;
-use atlas_core::ordering_protocol::networking::serialize::{OrderingProtocolMessage, PermissionedOrderingProtocolMessage, StatefulOrderProtocolMessage};
+use atlas_core::ordering_protocol::loggable::PersistentOrderProtocolTypes;
+use atlas_core::ordering_protocol::networking::serialize::{OrderingProtocolMessage, PermissionedOrderingProtocolMessage};
 use atlas_core::ordering_protocol::networking::signature_ver::OrderProtocolSignatureVerificationHelper;
-use atlas_core::persistent_log::PersistableOrderProtocol;
 use atlas_smr_application::serialize::ApplicationData;
 
+use crate::bft::log::decisions::{Proof, ProofMetadata};
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
-use crate::bft::msg_log::decisions::{DecisionLog, Proof, ProofMetadata};
 use crate::bft::sync::view::ViewInfo;
 
 #[cfg(feature = "serialize_capnp")]
@@ -120,7 +119,7 @@ impl<D> PBFTConsensus<D> where D: ApplicationData {
                         let header = pre_prepare.header();
                         let consensus_message = pre_prepare.message();
 
-                        Self::verify_consensus_message::<S, SV, NI>(network_info, header, consensus_message)
+                        Self::verify_consensus_message::<S, SV, NI>(network_info, header, consensus_message.consensus())
                     }).reduce(|a, b| a.and(b)).unwrap_or(Ok(true))
                 } else {
                     Ok(true)
@@ -140,10 +139,7 @@ impl<D> PBFTConsensus<D> where D: ApplicationData {
 
 impl<D> OrderingProtocolMessage<D> for PBFTConsensus<D>
     where D: ApplicationData, {
-
     type ProtocolMessage = PBFTMessage<D::Request>;
-    type LoggableMessage = ConsensusMessage<D::Request>;
-    type Proof = Proof<D::Request>;
     type ProofMetadata = ProofMetadata;
 
     fn verify_order_protocol_message<NI, OPVH>(network_info: &Arc<NI>, header: &Header, message: Self::ProtocolMessage) -> Result<(bool, Self::ProtocolMessage)> where NI: NetworkInformationProvider, OPVH: OrderProtocolSignatureVerificationHelper<D, Self, NI>, Self: Sized {
@@ -213,35 +209,6 @@ impl<D> OrderingProtocolMessage<D> for PBFTConsensus<D>
         }
     }
 
-    fn verify_proof<NI, OPVH>(network_info: &Arc<NI>, proof: Self::Proof) -> Result<(bool, Self::Proof)>
-        where NI: NetworkInformationProvider, OPVH: OrderProtocolSignatureVerificationHelper<D, Self, NI>, Self: Sized {
-        for pre_prepare in proof.pre_prepares() {
-            let (result, _) = OPVH::verify_protocol_message(network_info, pre_prepare.header(), PBFTMessage::Consensus(pre_prepare.message().clone()))?;
-
-            if !result {
-                return Ok((false, proof));
-            }
-        }
-
-        for prepare in proof.prepares() {
-            let (result, _) = OPVH::verify_protocol_message(network_info, prepare.header(), PBFTMessage::Consensus(prepare.message().clone()))?;
-
-            if !result {
-                return Ok((false, proof));
-            }
-        }
-
-        for commit in proof.commits() {
-            let (result, _) = OPVH::verify_protocol_message(network_info, commit.header(), PBFTMessage::Consensus(commit.message().clone()))?;
-
-            if !result {
-                return Ok((false, proof));
-            }
-        }
-
-        return Ok((true, proof));
-    }
-
     #[cfg(feature = "serialize_capnp")]
     fn serialize_capnp(builder: atlas_capnp::consensus_messages_capnp::protocol_message::Builder, msg: &Self::ProtocolMessage) -> Result<()> {
         capnp::serialize_message::<D>(builder, msg)
@@ -277,25 +244,39 @@ impl<D> PermissionedOrderingProtocolMessage for PBFTConsensus<D> where D: Applic
     type ViewInfo = ViewInfo;
 }
 
-impl<D, OP> StatefulOrderProtocolMessage<D, OP> for PBFTConsensus<D>
-    where D: ApplicationData + 'static {
-    type DecLog = DecisionLog<D::Request>;
+impl<D, OP> PersistentOrderProtocolTypes<D, OP> for PBFTConsensus<D>
+    where D: ApplicationData + 'static, OP: OrderingProtocolMessage<D> + 'static {
+
+    type Proof = Proof<D::Request>;
 
-    fn verify_decision_log<NI, OPVH>(network_info: &Arc<NI>, dec_log: Self::DecLog) -> Result<(bool, Self::DecLog)>
+    fn verify_proof<NI, OPVH>(network_info: &Arc<NI>, proof: Self::Proof) -> Result<(bool, Self::Proof)>
         where NI: NetworkInformationProvider,
-              D: ApplicationData,
-              OP: OrderingProtocolMessage<D>,
-              OPVH: OrderProtocolSignatureVerificationHelper<D, OP, NI>, {
-        Ok((false, dec_log))
-    }
+              OPVH: OrderProtocolSignatureVerificationHelper<D, Self, NI>,
+              Self: Sized {
+        for pre_prepare in proof.pre_prepares() {
+            let (result, _) = OPVH::verify_protocol_message(network_info, pre_prepare.header(), pre_prepare.message().clone())?;
 
-    #[cfg(feature = "serialize_capnp")]
-    fn serialize_declog_capnp(builder: atlas_capnp::cst_messages_capnp::dec_log::Builder, msg: &Self::DecLog) -> Result<()> {
-        todo!()
-    }
+            if !result {
+                return Ok((false, proof));
+            }
+        }
 
-    #[cfg(feature = "serialize_capnp")]
-    fn deserialize_declog_capnp(reader: atlas_capnp::cst_messages_capnp::dec_log::Reader) -> Result<Self::DecLog> {
-        todo!()
+        for prepare in proof.prepares() {
+            let (result, _) = OPVH::verify_protocol_message(network_info, prepare.header(), prepare.message().clone())?;
+
+            if !result {
+                return Ok((false, proof));
+            }
+        }
+
+        for commit in proof.commits() {
+            let (result, _) = OPVH::verify_protocol_message(network_info, commit.header(), commit.message().clone())?;
+
+            if !result {
+                return Ok((false, proof));
+            }
+        }
+
+        return Ok((true, proof));
     }
 }
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index b88bf3c0..74ab0901 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -21,7 +21,7 @@ use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::serialize::Serializable;
 use atlas_core::messages::{ClientRqInfo, Protocol};
 use atlas_core::ordering_protocol::{Decision, DecisionInfo, DecisionMetadata, DecisionsAhead, JoinInfo, OPExecResult, OPPollResult, OrderingProtocol, OrderingProtocolArgs, OrderProtocolTolerance, PermissionedOrderingProtocol, ProtocolConsensusDecision, ProtocolMessage, View};
-use atlas_core::ordering_protocol::loggable::{LoggableOrderProtocol, PProof};
+use atlas_core::ordering_protocol::loggable::{LoggableOrderProtocol, OrderProtocolPersistenceHelper, PProof};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::ordering_protocol::networking::serialize::{NetworkView, OrderingProtocolMessage};
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::{ReconfigurableOrderProtocol, ReconfigurationAttemptResult};
@@ -39,6 +39,7 @@ use crate::bft::config::PBFTConfig;
 use crate::bft::consensus::{Consensus, ConsensusPollStatus, ConsensusStatus, ProposerConsensusGuard};
 use crate::bft::log::decisions::{Proof, ProofMetadata};
 use crate::bft::log::{initialize_decided_log, Log};
+use crate::bft::log::decided::DecisionLog;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, ObserveEventKind, PBFTMessage, ViewChangeMessage};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{CONSENSUS_INSTALL_STATE_TIME_ID, MSG_LOG_INSTALL_TIME_ID};
@@ -175,7 +176,7 @@ impl<D, NT> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT>
         Ok(())
     }
 
-    fn poll(&mut self) -> OPPollResult<ProofMetadata, PBFTMessage<D::Request>, D::Request> {
+    fn poll(&mut self) -> Result<OPPollResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>> {
         trace!("{:?} // Polling {:?}", self.node.id(), self.phase);
 
         match self.phase {
@@ -284,7 +285,7 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
                                                     SeqNo::ZERO, watermark, consensus_guard.clone(),
                                                     timeouts.clone());
 
-        let dec_log = initialize_decided_log::<D>(node_id)?;
+        let dec_log = initialize_decided_log::<D>(node_id);
 
         let proposer = Proposer::<D, NT>::new(node.clone(), batch_input, sync.clone(), timeouts.clone(),
                                               executor.clone(), consensus_guard.clone(),
@@ -323,7 +324,7 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
         Ok(replica)
     }
 
-    fn poll_sync_phase(&mut self) -> OPPollResult<ProofMetadata, PBFTMessage<D::Request>, D::Request> {
+    fn poll_sync_phase(&mut self) -> Result<OPPollResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>> {
 
         // retrieve a view change message to be processed
         let poll_result = self.synchronizer.poll();
@@ -331,9 +332,9 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
         debug!("{:?} // Polling sync phase {:?}", self.node.id(), poll_result);
 
         match poll_result {
-            SynchronizerPollStatus::Recv => OPPollResult::ReceiveMsg,
+            SynchronizerPollStatus::Recv => Ok(OPPollResult::ReceiveMsg),
             SynchronizerPollStatus::NextMessage(message) => {
-                OPPollResult::Exec(message)
+                Ok(OPPollResult::Exec(message))
             }
             SynchronizerPollStatus::ResumeViewChange => {
                 debug!("{:?} // Resuming view change", self.node.id());
@@ -357,42 +358,44 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
                             let joined = JoinInfo::new(node, self.synchronizer.view().quorum_members().clone());
 
                             if decisions.is_empty() {
-                                OPExecResult::QuorumJoined(DecisionsAhead::ClearAhead, None, joined)
+                                Ok(OPPollResult::QuorumJoined(DecisionsAhead::ClearAhead, None, joined))
                             } else {
-                                OPExecResult::QuorumJoined(DecisionsAhead::ClearAhead, Some(decisions), joined)
+                                Ok(OPPollResult::QuorumJoined(DecisionsAhead::ClearAhead, Some(decisions), joined))
                             }
                         }
                         SynchronizerStatus::NewView(consensus_status, decisions) => {
                             let decisions = self.handle_sync_result(consensus_status, decisions)?;
 
-                            OPExecResult::ProgressedDecision(DecisionsAhead::ClearAhead, decisions)
+                            Ok(OPPollResult::ProgressedDecision(DecisionsAhead::ClearAhead, decisions))
                         }
                         _ => {
-                            warn!("Received sync status that is not handled")
+                            warn!("Received sync status that is not handled");
+
+                            Ok(OPPollResult::RePoll)
                         }
                     }
+                } else {
+                    Ok(OPPollResult::RePoll)
                 }
-
-                OPPollResult::RePoll
             }
         }
     }
 
-    fn poll_normal_phase(&mut self) -> OPPollResult<ProofMetadata, PBFTMessage<D::Request>, D::Request> {
+    fn poll_normal_phase(&mut self) -> Result<OPPollResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>> {
         // check if we have STOP messages to be processed,
         // and update our phase when we start installing
         // the new view
         // Consume them until we reached a point where no new messages can be processed
         while self.synchronizer.can_process_stops() {
-            let sync_protocol = self.poll_sync_phase();
+            let sync_protocol = self.poll_sync_phase()?;
 
             if let OPPollResult::Exec(s_message) = sync_protocol {
                 let (header, message) = (s_message.header(), s_message.message());
 
-                if let PBFTMessage::ViewChange(view_change) = message.into_inner() {
+                if let PBFTMessage::ViewChange(view_change) = message {
                     let result = self.adv_sync(s_message);
 
-                    return match result {
+                    return Ok(match result {
                         SyncPhaseRes::RunSyncProtocol => {
                             self.switch_phase(ConsensusPhase::SyncPhase);
 
@@ -415,7 +418,7 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
                             error!("Polling the sync phase should never return anything other than a run sync protocol or run cst protocol message, Protocol Finished");
                             OPPollResult::RePoll
                         }
-                    };
+                    });
                 } else {
                     // The synchronizer should never return anything other than a view
                     // change message
@@ -430,19 +433,19 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
         // `TboQueue`, in the consensus module.
         let polled_message = self.consensus.poll();
 
-        match polled_message {
+        Ok(match polled_message {
             ConsensusPollStatus::Recv => OPPollResult::ReceiveMsg,
             ConsensusPollStatus::NextMessage(message) => {
                 OPPollResult::Exec(message)
             }
             ConsensusPollStatus::Decided(decisions) => {
-                let finalized_decisions = self.finalize_all_possible().expect("Failed to finalize decisions");
+                let finalized_decisions = self.finalize_all_possible()?;
 
                 let decisions = self.merge_decisions(decisions, finalized_decisions)?;
 
-                OPPollResult::ProgressedDecision(decisions)
+                OPPollResult::ProgressedDecision(DecisionsAhead::Ignore, decisions)
             }
-        }
+        })
     }
 
     fn update_sync_phase(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>> {
@@ -771,11 +774,7 @@ const CF_PRE_PREPARES: &str = "PRE_PREPARES";
 const CF_PREPARES: &str = "PREPARES";
 const CF_COMMIT: &str = "COMMITS";
 
-impl<D, NT> LoggableOrderProtocol<D, NT> for PBFTOrderProtocol<D, NT>
-    where D: ApplicationData + 'static,
-          NT: OrderProtocolSendNode<D, PBFT<D>> {
-    type PersistableTypes = ();
-
+impl<D, NT> OrderProtocolPersistenceHelper<D, PBFTConsensus<D>, PBFTConsensus<D>> for PBFTOrderProtocol<D, NT> where D: 'static + ApplicationData, NT: OrderProtocolSendNode<D, PBFT<D>> {
     fn message_types() -> Vec<&'static str> {
         vec![
             CF_PRE_PREPARES,
@@ -799,10 +798,8 @@ impl<D, NT> LoggableOrderProtocol<D, NT> for PBFTOrderProtocol<D, NT>
                     }
                 }
             }
-            PBFTMessage::ViewChange(view_change) => {
-                Err(Error::simple_with_msg(ErrorKind::Consensus, "Failed to get type for view change message."))
-            }
-            PBFTMessage::ObserverMessage(_) => {}
+            PBFTMessage::ViewChange(view_change) => Err(Error::simple_with_msg(ErrorKind::Consensus, "Failed to get type for view change message.")),
+            PBFTMessage::ObserverMessage(_) => Err(Error::simple_with_msg(ErrorKind::Consensus, "Failed to get type for view change message."))
         }
     }
 
@@ -812,7 +809,7 @@ impl<D, NT> LoggableOrderProtocol<D, NT> for PBFTOrderProtocol<D, NT>
         let mut commits = Vec::with_capacity(messages.len() / 2);
 
         for message in messages {
-            match message.message().kind() {
+            match message.message().consensus().kind() {
                 ConsensusMessageKind::PrePrepare(_) => {
                     pre_prepares.push(Arc::new(ReadOnly::new(message)));
                 }
@@ -828,14 +825,14 @@ impl<D, NT> LoggableOrderProtocol<D, NT> for PBFTOrderProtocol<D, NT>
         Proof::new(metadata, pre_prepares, prepares, commits)
     }
 
-    fn init_proof_from_scm(metadata: DecisionMetadata<D, Self::Serialization>,
-                           messages: Vec<ShareableConsensusMessage<D, Self::Serialization>>) -> PProof<D, Self::Serialization, Self::PersistableTypes> {
+    fn init_proof_from_scm(metadata: DecisionMetadata<D, PBFTConsensus<D>>,
+                           messages: Vec<ShareableConsensusMessage<D, PBFTConsensus<D>>>) -> PProof<D, PBFTConsensus<D>, PBFTConsensus<D>> {
         let mut pre_prepares = Vec::with_capacity(messages.len() / 2);
         let mut prepares = Vec::with_capacity(messages.len() / 2);
         let mut commits = Vec::with_capacity(messages.len() / 2);
 
         for message in messages {
-            match message.message().kind() {
+            match message.message().consensus().kind() {
                 ConsensusMessageKind::PrePrepare(_) => {
                     pre_prepares.push(message);
                 }
@@ -851,29 +848,35 @@ impl<D, NT> LoggableOrderProtocol<D, NT> for PBFTOrderProtocol<D, NT>
         Proof::new(metadata, pre_prepares, prepares, commits)
     }
 
-    fn decompose_proof(proof: &Proof<D::Request>) -> (&ProofMetadata, Vec<&ShareableMessage<PBFTMessage<D::Request>>>) {
+    fn decompose_proof(proof: &Proof<D::Request>) -> (&ProofMetadata, Vec<&StoredMessage<PBFTMessage<D::Request>>>) {
         let mut messages = Vec::new();
 
         for message in proof.pre_prepares() {
-            messages.push(message);
+            messages.push(&***message);
         }
 
         for message in proof.prepares() {
-            messages.push(message);
+            messages.push(&***message);
         }
 
         for message in proof.commits() {
-            messages.push(message);
+            messages.push(&***message);
         }
 
         (proof.metadata(), messages)
     }
 
-    fn get_requests_in_proof(proof: &PProof<D, Self::Serialization, Self::PersistableTypes>) -> Result<ProtocolConsensusDecision<D::Request>> {
+    fn get_requests_in_proof(proof: &PProof<D, PBFTConsensus<D>, PBFTConsensus<D>>) -> Result<ProtocolConsensusDecision<D::Request>> {
         Ok(ProtocolConsensusDecision::from(proof))
     }
 }
 
+impl<D, NT> LoggableOrderProtocol<D, NT> for PBFTOrderProtocol<D, NT>
+    where D: ApplicationData + 'static,
+          NT: OrderProtocolSendNode<D, PBFT<D>> {
+    type PersistableTypes = PBFTConsensus<D>;
+}
+
 impl<D, NT, RP> ReconfigurableOrderProtocol<RP> for PBFTOrderProtocol<D, NT>
     where D: ApplicationData + 'static,
           RP: ReconfigurationProtocolMessage + 'static,
diff --git a/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
index b68d2b9a..86e37f02 100644
--- a/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/follower_sync/mod.rs
@@ -2,9 +2,9 @@ use std::{marker::PhantomData};
 use atlas_common::ordering::Orderable;
 use atlas_core::messages::ClientRqInfo;
 use atlas_smr_application::serialize::ApplicationData;
+use crate::bft::log::decisions::StoredConsensusMessage;
 
-use crate::bft::message::{ConsensusMessageKind};
-use crate::bft::msg_log::decisions::StoredConsensusMessage;
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind};
 
 pub struct FollowerSynchronizer<D: ApplicationData> {
     _phantom: PhantomData<D>,
@@ -20,9 +20,9 @@ impl<D: ApplicationData + 'static> FollowerSynchronizer<D> {
     /// proposed, they won't timeout
     pub fn watch_request_batch(
         &self,
-        pre_prepare: &StoredConsensusMessage<D::Request>,
+        pre_prepare: &ConsensusMessage<D::Request>,
     ) -> Vec<ClientRqInfo> {
-        let requests = match pre_prepare.message().kind() {
+        let requests = match pre_prepare.kind() {
             ConsensusMessageKind::PrePrepare(req) => { req }
             _ => { panic!() }
         };
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index abe716bd..eb16d1a6 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -56,10 +56,9 @@ macro_rules! extract_msg {
     };
 
     ($t:ty => $opt:block, $g:expr, $q:expr) => {
-        if let Some(stored) = tbo_pop_message::<StoredMessage<ViewChangeMessage<$t>>>($q) {
+        if let Some(stored) = tbo_pop_message::<ShareableMessage<PBFTMessage<$t>>>($q) {
             $opt
-            let (header, message) = stored.into_inner();
-            SynchronizerPollStatus::NextMessage(header, message)
+            SynchronizerPollStatus::NextMessage(stored)
         } else {
             *$g = false;
             SynchronizerPollStatus::Recv
@@ -680,7 +679,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     /// Advances the state of the view change state machine.
     //
     // TODO: retransmit STOP msgs
-    pub fn process_message<NT, PL>(
+    pub fn process_message<NT>(
         &self,
         s_message: ShareableMessage<PBFTMessage<D::Request>>,
         timeouts: &Timeouts,
@@ -690,7 +689,6 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         node: &Arc<NT>,
     ) -> SynchronizerStatus<D::Request>
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
-              PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
 
         let (header, message) = (s_message.header(), s_message.message().view_change());
@@ -1353,9 +1351,8 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
     /// Start the quorum join procedure to integrate the given joining node into the current quorum
     /// of the system
-    pub fn start_join_quorum<NT, PL>(&self, joining_node: NodeId, node: &NT, timeouts: &Timeouts, log: &Log<D>) -> SyncReconfigurationResult
-        where NT: OrderProtocolSendNode<D, PBFT<D>>,
-              PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+    pub fn start_join_quorum<NT>(&self, joining_node: NodeId, node: &NT, timeouts: &Timeouts, log: &Log<D>) -> SyncReconfigurationResult
+        where NT: OrderProtocolSendNode<D, PBFT<D>>, {
         let current_view = self.view();
 
         info!("{:?} // Starting the quorum join procedure for node {:?}", node.id(), joining_node);
@@ -1446,13 +1443,12 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     }
 
     /// Trigger a view change locally
-    pub fn begin_quorum_view_change<NT, PL>(&self,
+    pub fn begin_quorum_view_change<NT>(&self,
                                             join_cert: Option<NodeId>,
                                             node: &NT,
                                             timeouts: &Timeouts,
                                             _log: &Log<D>, )
         where NT: OrderProtocolSendNode<D, PBFT<D>>,
-              PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
         debug!("Beginning quorum view change with certificate {} at phase {:?}",  join_cert.is_some(), self.phase.get());
 
@@ -1501,14 +1497,13 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     /// that have timed out on the current replica.
     /// If the timed out requests are None, that means that the view change
     /// originated in the other replicas.
-    pub fn begin_view_change<NT, PL>(
+    pub fn begin_view_change<NT>(
         &self,
         timed_out: Option<Vec<StoredRequestMessage<D::Request>>>,
         node: &NT,
         timeouts: &Timeouts,
         _log: &Log<D>,
     ) where NT: OrderProtocolSendNode<D, PBFT<D>>,
-            PL: OrderingProtocolLog<D, PBFTConsensus<D>>
     {
         match (self.phase.get(), &timed_out) {
             // we have received STOP messages from peer nodes,
@@ -1655,7 +1650,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         };
 
         // finalize view change by broadcasting a PREPARE msg
-        let consensus_result = consensus.finalize_view_change((header, message), &view, self, timeouts, log, node)?;
+        let consensus_result = consensus.finalize_view_change((header, message), &view, self, timeouts, log, node).expect("Failed to finalize view change in consensus");
 
         // Update proto phase
         self.phase.replace(ProtoPhase::Init);
@@ -1677,12 +1672,13 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     /// proposed, they won't timeout
     pub fn request_batch_received(
         &self,
+        header: &Header,
         pre_prepare: &ConsensusMessage<D::Request>,
         timeouts: &Timeouts,
     ) -> Vec<ClientRqInfo> {
         match &self.accessory {
             SynchronizerAccessory::Replica(rep) => {
-                rep.received_request_batch(pre_prepare, timeouts)
+                rep.received_request_batch(header, pre_prepare, timeouts)
             }
             SynchronizerAccessory::Follower(fol) => fol.watch_request_batch(pre_prepare),
         }
@@ -2045,7 +2041,7 @@ fn highest_proof<'a, D, I, NT>(
                 .commits()
                 .iter()
                 .filter(|stored| {
-                    stored.message()
+                    stored.message().consensus()
                         .has_proposed_digest(&digest)
                         //If he does not have the digest, then it is not valid
                         .unwrap_or(false)
@@ -2059,7 +2055,7 @@ fn highest_proof<'a, D, I, NT>(
                 .iter()
                 .filter(|stored| {
                     stored
-                        .message()
+                        .message().consensus()
                         .has_proposed_digest(&digest)
                         //If he does not have the digest, then it is not valid
                         .unwrap_or(false)
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 54d6e869..3fba7c9d 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -22,11 +22,11 @@ use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::{metric_duration, metric_increment};
 
 use crate::bft::consensus::Consensus;
+use crate::bft::log::decisions::CollectData;
+use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
 use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{SYNC_BATCH_RECEIVED_ID, SYNC_STOPPED_COUNT_ID, SYNC_STOPPED_REQUESTS_ID, SYNC_WATCH_REQUESTS_ID};
-use crate::bft::msg_log::decided_log::Log;
-use crate::bft::msg_log::decisions::{CollectData, StoredConsensusMessage};
 use crate::bft::PBFT;
 use crate::bft::sync::view::ViewInfo;
 
@@ -57,17 +57,16 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     ///
     /// Therefore, we start by clearing our stopped requests and treating them as
     /// newly proposed requests (by resetting their timer)
-    pub(super) fn handle_stopping_quorum<NT, PL>(
+    pub(super) fn handle_stopping_quorum<NT>(
         &self,
         base_sync: &Synchronizer<D>,
         previous_view: ViewInfo,
-        consensus: &Consensus<D, PL>,
-        log: &Log<D, PL>,
+        consensus: &Consensus<D>,
+        log: &Log<D>,
         pre_processor: &RequestPreProcessor<D::Request>,
         timeouts: &Timeouts,
         node: &NT,
-    ) where NT: OrderProtocolSendNode<D, PBFT<D>>,
-            PL: OrderingProtocolLog<D, PBFTConsensus<D>> {
+    ) where NT: OrderProtocolSendNode<D, PBFT<D>>{
         // NOTE:
         // - install new view (i.e. update view seq no) (Done in the synchronizer)
         // - add requests from STOP into client requests
@@ -83,10 +82,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         let current_view_seq = view_info.sequence_number();
         let current_leader = view_info.leader();
 
-        let last_proof = log
-            //we use the previous views' f because the new view could have changed
-            //The N of the network (With reconfigurable views)
-            .last_proof(previous_view.params().f());
+        let last_proof = log.last_proof();
 
         let incomplete_proof = consensus.collect_incomplete_proof(previous_view.params().f());
 
@@ -178,6 +174,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
     /// proposed, they won't timeout
     pub fn received_request_batch(
         &self,
+        header: &Header,
         pre_prepare: &ConsensusMessage<D::Request>,
         timeouts: &Timeouts,
     ) -> Vec<ClientRqInfo> {
@@ -195,7 +192,7 @@ impl<D: ApplicationData + 'static> ReplicaSynchronizer<D> {
         let mut timeout_info = Vec::with_capacity(requests.len());
         let mut digests = Vec::with_capacity(requests.len());
 
-        let sending_node = pre_prepare.header().from();
+        let sending_node = header.from();
 
         for x in requests {
             let header = x.header();
diff --git a/febft-pbft-consensus/src/bft/sync/view/mod.rs b/febft-pbft-consensus/src/bft/sync/view/mod.rs
index b329e7d4..924a2030 100644
--- a/febft-pbft-consensus/src/bft/sync/view/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/view/mod.rs
@@ -13,7 +13,7 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
 use atlas_core::ordering_protocol::networking::serialize::NetworkView;
-use atlas_execution::system_params::SystemParams;
+use atlas_common::system_params::SystemParams;
 
 /// This struct contains information related with an
 /// active `febft` view.
diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 7e1fe874..c59e987a 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -27,9 +27,9 @@ use atlas_core::state_transfer::{Checkpoint, CstM, StateTransferProtocol, STResu
 use atlas_core::state_transfer::monolithic_state::MonolithicStateTransfer;
 use atlas_core::state_transfer::networking::StateTransferSendNode;
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, Timeouts};
-use atlas_execution::app::Application;
-use atlas_execution::serialize::ApplicationData;
-use atlas_execution::state::monolithic_state::{InstallStateMessage, MonolithicState};
+use atlas_smr_application::app::Application;
+use atlas_smr_application::serialize::ApplicationData;
+use atlas_smr_application::state::monolithic_state::{InstallStateMessage, MonolithicState};
 use atlas_metrics::metrics::metric_duration;
 
 use crate::config::StateTransferConfig;
diff --git a/febft-state-transfer/src/message/serialize/capnp/mod.rs b/febft-state-transfer/src/message/serialize/capnp/mod.rs
index e294ed2b..8d274007 100644
--- a/febft-state-transfer/src/message/serialize/capnp/mod.rs
+++ b/febft-state-transfer/src/message/serialize/capnp/mod.rs
@@ -1,6 +1,6 @@
-use atlas_execution::serialize::ApplicationData;
+use atlas_smr_application::serialize::ApplicationData;
 use atlas_common::error::*;
-use atlas_execution::state::divisible_state::DivisibleState;
+use atlas_smr_application::state::divisible_state::DivisibleState;
 use crate::message::CstMessage;
 
 fn serialize_state_transfer<S>(mut state_transfer: atlas_capnp::cst_messages_capnp::cst_message::Builder,
diff --git a/febft-state-transfer/src/message/serialize/mod.rs b/febft-state-transfer/src/message/serialize/mod.rs
index a19f146f..265f6be1 100644
--- a/febft-state-transfer/src/message/serialize/mod.rs
+++ b/febft-state-transfer/src/message/serialize/mod.rs
@@ -5,7 +5,7 @@ use atlas_communication::message::Header;
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::state_transfer::networking::serialize::StateTransferMessage;
 use atlas_core::state_transfer::networking::signature_ver::StateTransferVerificationHelper;
-use atlas_execution::state::monolithic_state::MonolithicState;
+use atlas_smr_application::state::monolithic_state::MonolithicState;
 
 use crate::message::CstMessage;
 

From f15dfb86af5fffdc9d0981d82b7b8ec4ebe399b7 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Sun, 15 Oct 2023 22:22:30 +0100
Subject: [PATCH 62/80] Fixed log transfer compilation issues.

---
 febft-pbft-consensus/src/bft/message/serialize/mod.rs | 5 ++---
 febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs | 1 +
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index a8a7d506..4bbd99f0 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -244,9 +244,8 @@ impl<D> PermissionedOrderingProtocolMessage for PBFTConsensus<D> where D: Applic
     type ViewInfo = ViewInfo;
 }
 
-impl<D, OP> PersistentOrderProtocolTypes<D, OP> for PBFTConsensus<D>
-    where D: ApplicationData + 'static, OP: OrderingProtocolMessage<D> + 'static {
-
+impl<D> PersistentOrderProtocolTypes<D, Self> for PBFTConsensus<D>
+    where D: ApplicationData + 'static {
     type Proof = Proof<D::Request>;
 
     fn verify_proof<NI, OPVH>(network_info: &Arc<NI>, proof: Self::Proof) -> Result<(bool, Self::Proof)>
diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 3fba7c9d..73ce87ca 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -12,6 +12,7 @@ use log::{debug, error, info};
 use atlas_common::collections;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::Orderable;
+use atlas_communication::message::Header;
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;

From 3703488982aabbc3144c9f6771cc9c9af4ae656f Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 16 Oct 2023 18:42:39 +0100
Subject: [PATCH 63/80] Fixed persistent log compilation. Just missing a couple
 of bugs with the pbft consensus

---
 febft-pbft-consensus/src/bft/consensus/mod.rs | 57 +------------------
 febft-pbft-consensus/src/bft/sync/mod.rs      | 25 ++++----
 2 files changed, 14 insertions(+), 68 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index aea07cb9..a613ab86 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -36,7 +36,7 @@ use crate::bft::sync::view::ViewInfo;
 pub mod decision;
 pub mod accessory;
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 /// Status returned from processing a consensus message.
 pub enum ConsensusStatus<O> {
     /// A particular node tried voting twice.
@@ -57,7 +57,7 @@ pub enum ConsensusStatus<O> {
     Decided(MaybeVec<OPDecision<O>>),
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 /// Represents the status of calling `poll()` on a `Consensus`.
 pub enum ConsensusPollStatus<O> {
     /// The `Replica` associated with this `Consensus` should
@@ -522,59 +522,6 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
         decision
     }
 
-    /// Install the received state into the consensus
-    pub fn install_state(&mut self,
-                         view_info: ViewInfo,
-                         dec_log: &DecisionLog<D::Request>) -> Result<(Vec<D::Request>)> {
-
-        // get the latest seq no
-        let seq_no = {
-            let last_exec = dec_log.last_execution();
-            if last_exec.is_none() {
-                self.sequence_number()
-            } else {
-                last_exec.unwrap()
-            }
-        };
-
-        if seq_no > SeqNo::ZERO {
-            // If we have installed a new state, then we must be recovering and therefore should
-            // Stop timeouts
-            self.is_recovering = true;
-        }
-
-        // skip old messages
-        self.install_sequence_number(seq_no.next(), &view_info);
-
-        // Update the decisions with the new view information
-        self.install_view(&view_info);
-
-        let mut reqs = Vec::with_capacity(dec_log.proofs().len());
-
-        for proof in dec_log.proofs() {
-            if !proof.are_pre_prepares_ordered()? {
-                unreachable!()
-            }
-
-            for pre_prepare in proof.pre_prepares() {
-                let x: &ConsensusMessage<D::Request> = pre_prepare.message();
-
-                match x.kind() {
-                    ConsensusMessageKind::PrePrepare(pre_prepare_reqs) => {
-                        for req in pre_prepare_reqs {
-                            let rq_msg: &RequestMessage<D::Request> = req.message();
-
-                            reqs.push(rq_msg.operation().clone());
-                        }
-                    }
-                    _ => { unreachable!() }
-                }
-            }
-        }
-
-        Ok(reqs)
-    }
-
     pub fn install_sequence_number(&mut self, novel_seq_no: SeqNo, view: &ViewInfo) {
         info!("{:?} // Installing sequence number {:?} vs current {:?}", self.node_id, novel_seq_no, self.seq_no);
 
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index eb16d1a6..505e78b6 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -117,7 +117,7 @@ pub struct LeaderCollects<O> {
     // Done
     proposed: FwdConsensusMessage<O>,
     // The collect messages the leader has received.
-    collects: Vec<StoredMessage<ViewChangeMessage<O>>>,
+    collects: Vec<ShareableMessage<PBFTMessage<O>>>,
 }
 
 impl<O> LeaderCollects<O> {
@@ -125,7 +125,7 @@ impl<O> LeaderCollects<O> {
         &self.proposed
     }
 
-    pub fn collects(&self) -> &Vec<StoredMessage<ViewChangeMessage<O>>> {
+    pub fn collects(&self) -> &Vec<ShareableMessage<PBFTMessage<O>>> {
         &self.collects
     }
 
@@ -134,7 +134,7 @@ impl<O> LeaderCollects<O> {
         self,
     ) -> (
         FwdConsensusMessage<O>,
-        Vec<StoredMessage<ViewChangeMessage<O>>>,
+        Vec<ShareableMessage<PBFTMessage<O>>>,
     ) {
         (self.proposed, self.collects)
     }
@@ -1170,8 +1170,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
                             let fwd_request = FwdConsensusMessage::new(header, message);
 
-                            let collects = collects_guard.values()
-                                .cloned().collect();
+                            let collects = collects_guard.values().cloned().collect();
 
                             let message = PBFTMessage::ViewChange(ViewChangeMessage::new(
                                 next_view.sequence_number(),
@@ -1733,7 +1732,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 // may be executing the same CID when there is a leader change
     #[inline]
     fn normalized_collects<'a>(
-        collects: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request>>>,
+        collects: &'a IntMap<ShareableMessage<PBFTMessage<D::Request>>>,
         in_exec: SeqNo,
     ) -> impl Iterator<Item=Option<&'a CollectData<D::Request>>> {
         let values = collects.values();
@@ -1746,7 +1745,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     // TODO: quorum sizes may differ when we implement reconfiguration
     #[inline]
     fn highest_proof<'a, NT>(
-        guard: &'a IntMap<StoredMessage<ViewChangeMessage<D::Request>>>,
+        guard: &'a IntMap<ShareableMessage<PBFTMessage<D::Request>>>,
         view: &ViewInfo,
         node: &NT,
     ) -> Option<&'a Proof<D::Request>>
@@ -1956,9 +1955,9 @@ fn certified_value<O>(
 }
 
 fn collect_data<'a, O: 'a>(
-    collects: impl Iterator<Item=&'a StoredMessage<ViewChangeMessage<O>>>,
+    collects: impl Iterator<Item=&'a ShareableMessage<PBFTMessage<O>>>,
 ) -> impl Iterator<Item=&'a CollectData<O>> {
-    collects.filter_map(|stored| match stored.message().kind() {
+    collects.filter_map(|stored| match stored.message().view_change().kind() {
         ViewChangeMessageKind::StopData(collects) => Some(collects),
         _ => None,
     })
@@ -1979,14 +1978,14 @@ fn normalized_collects<'a, O: 'a>(
 
 fn signed_collects<D, NT>(
     node: &NT,
-    collects: Vec<StoredMessage<ViewChangeMessage<D::Request>>>,
-) -> Vec<StoredMessage<ViewChangeMessage<D::Request>>>
+    collects: Vec<ShareableMessage<PBFTMessage<D::Request>>>,
+) -> Vec<ShareableMessage<PBFTMessage<D::Request>>>
     where D: ApplicationData + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>>
 {
     collects
         .into_iter()
-        .filter(|stored| validate_signature::<D, _, _>(node, stored))
+        .filter(|stored| validate_signature::<D, _, _>(node, &**stored))
         .collect()
 }
 
@@ -2026,7 +2025,7 @@ fn highest_proof<'a, D, I, NT>(
 ) -> Option<&'a Proof<D::Request>>
     where
         D: ApplicationData + 'static,
-        I: Iterator<Item=&'a StoredMessage<ViewChangeMessage<D::Request>>>,
+        I: Iterator<Item=&'a ShareableMessage<PBFTMessage<D::Request>>>,
         NT: OrderProtocolSendNode<D, PBFT<D>>
 {
     collect_data(collects)

From 4762238845942a837f4ad6743b87a7cf921c5d42 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Tue, 17 Oct 2023 16:58:45 +0100
Subject: [PATCH 64/80] Fixed febft compilation errors.

---
 febft-pbft-consensus/src/bft/consensus/mod.rs |  11 +-
 .../src/bft/log/decided/mod.rs                |   7 +-
 .../src/bft/log/deciding/mod.rs               |  38 ++--
 .../src/bft/log/decisions/mod.rs              |  36 +++-
 febft-pbft-consensus/src/bft/log/mod.rs       |   3 +-
 .../src/bft/message/serialize/mod.rs          | 203 +++++++-----------
 febft-pbft-consensus/src/bft/sync/mod.rs      |  88 +++++---
 .../src/message/serialize/mod.rs              |   2 +-
 8 files changed, 199 insertions(+), 189 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index a613ab86..4522c6ed 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -11,7 +11,7 @@ use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::maybe_vec::MaybeVec;
 use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_advance_message_queue_return, tbo_queue_message};
+use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_advance_message_queue_return, tbo_queue_message, tbo_queue_message_arc};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage};
 use atlas_core::ordering_protocol::Decision;
@@ -149,23 +149,23 @@ impl<O> TboQueue<O> {
     /// Queues a `PRE-PREPARE` message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
     fn queue_pre_prepare(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
-        tbo_queue_message(
+        tbo_queue_message_arc(
             self.base_seq(),
             &mut self.pre_prepares,
-            message,
+            (message.sequence_number(), message),
         )
     }
 
     /// Queues a `PREPARE` message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
     fn queue_prepare(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
-        tbo_queue_message(self.base_seq(), &mut self.prepares, message)
+        tbo_queue_message_arc(self.base_seq(), &mut self.prepares, (message.sequence_number(), message))
     }
 
     /// Queues a `COMMIT` message for later processing, or drops it
     /// immediately if it pertains to an older consensus instance.
     fn queue_commit(&mut self, message: ShareableMessage<PBFTMessage<O>>) {
-        tbo_queue_message(self.base_seq(), &mut self.commits, message)
+        tbo_queue_message_arc(self.base_seq(), &mut self.commits, (message.sequence_number(), message))
     }
 
     /// Clear this queue
@@ -440,7 +440,6 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
             }
             DecisionStatus::DecidedIgnored => ConsensusStatus::Decided(MaybeVec::None),
             DecisionStatus::MessageIgnored => ConsensusStatus::MessageIgnored
-
         })
     }
 
diff --git a/febft-pbft-consensus/src/bft/log/decided/mod.rs b/febft-pbft-consensus/src/bft/log/decided/mod.rs
index e10092a3..93ff14ef 100644
--- a/febft-pbft-consensus/src/bft/log/decided/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decided/mod.rs
@@ -1,6 +1,5 @@
 use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_common::error::*;
-use crate::bft::log::deciding::CompletedBatch;
+
 use crate::bft::log::decisions::Proof;
 
 /// A necessary decision log for the ability to perform view changes.
@@ -29,7 +28,7 @@ impl<O> DecisionLog<O> {
     }
 
     pub fn last_execution(&self) -> Option<SeqNo> {
-        self.last_decision.map(|decision| decision.sequence_number())
+        self.last_decision.as_ref().map(|decision| decision.sequence_number())
     }
 
     pub fn append_proof(&mut self, proof: Proof<O>) {
@@ -41,6 +40,6 @@ impl<O> DecisionLog<O> {
 
 impl<O> Orderable for DecisionLog<O> {
     fn sequence_number(&self) -> SeqNo {
-        self.last_decision.map(|f| f.sequence_number()).unwrap_or(SeqNo::ZERO)
+        self.last_decision.as_ref().map(|f| f.sequence_number()).unwrap_or(SeqNo::ZERO)
     }
 }
diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
index 4e49f14e..5c22eb09 100644
--- a/febft-pbft-consensus/src/bft/log/deciding/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -21,32 +21,32 @@ use crate::bft::sync::view::ViewInfo;
 pub struct MessageLog<O> {
     pre_prepare: Vec<Option<ShareableMessage<PBFTMessage<O>>>>,
     prepares: Vec<ShareableMessage<PBFTMessage<O>>>,
-    commits: Vec<ShareableMessage<PBFTMessage<O>>>
+    commits: Vec<ShareableMessage<PBFTMessage<O>>>,
 }
 
 pub struct FinishedMessageLog<O> {
-    pre_prepares: Vec<ShareableMessage<PBFTMessage<O>>>,
-    prepares: Vec<ShareableMessage<PBFTMessage<O>>>,
-    commits: Vec<ShareableMessage<PBFTMessage<O>>>
+    pub(super) pre_prepares: Vec<ShareableMessage<PBFTMessage<O>>>,
+    pub(super) prepares: Vec<ShareableMessage<PBFTMessage<O>>>,
+    pub(super) commits: Vec<ShareableMessage<PBFTMessage<O>>>,
 }
 
 /// Information about the completed batch, the contained requests and
 /// other relevant information
 pub struct CompletedBatch<O> {
-    seq: SeqNo,
+    pub(super) seq: SeqNo,
     // The overall digest of the entire batch
-    digest: Digest,
+    pub(super) digest: Digest,
     // The ordering of the pre prepare requests
-    pre_prepare_ordering: Vec<Digest>,
+    pub(super) pre_prepare_ordering: Vec<Digest>,
     // The messages that are a part of this decision
-    contained_messages: FinishedMessageLog<O>,
+    pub(super) contained_messages: FinishedMessageLog<O>,
     // The information of the client requests that are contained in this batch
-    client_request_info: Vec<ClientRqInfo>,
+    pub(super) client_request_info: Vec<ClientRqInfo>,
     // The client requests contained in this batch
-    client_requests: Vec<StoredRequestMessage<O>>,
+    pub(super) client_requests: Vec<StoredRequestMessage<O>>,
 
     // The metadata for the batch
-    batch_meta: BatchMeta,
+    pub(super) batch_meta: BatchMeta,
 }
 
 pub struct WorkingDecisionLog<O> {
@@ -292,7 +292,7 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
             contained_messages: self.message_log.finalize(),
             client_request_info: self.client_rqs,
             batch_meta,
-            client_requests: requests
+            client_requests: requests,
         })
     }
 }
@@ -304,11 +304,25 @@ impl<O> Orderable for CompletedBatch<O> {
 }
 
 impl<O> CompletedBatch<O> {
+    pub fn new(seq: SeqNo, digest: Digest, pre_prepare_ordering: Vec<Digest>,
+               contained_messages: FinishedMessageLog<O>, client_request_info: Vec<ClientRqInfo>,
+               client_requests: Vec<StoredRequestMessage<O>>, batch_meta: BatchMeta) -> Self {
+        Self {
+            seq,
+            digest,
+            pre_prepare_ordering,
+            contained_messages,
+            client_request_info,
+            client_requests,
+            batch_meta,
+        }
+    }
 
     pub fn request_count(&self) -> usize {
         self.client_requests.len()
     }
 
+
 }
 
 impl<O> Orderable for WorkingDecisionLog<O> {
diff --git a/febft-pbft-consensus/src/bft/log/decisions/mod.rs b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
index 5d0a62c9..1fedd0ea 100644
--- a/febft-pbft-consensus/src/bft/log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
@@ -10,7 +10,7 @@ use atlas_common::ordering::{Orderable, SeqNo};
 use atlas_core::ordering_protocol::networking::serialize::OrderProtocolProof;
 use atlas_core::smr::smr_decision_log::ShareableMessage;
 
-use crate::bft::message::PBFTMessage;
+use crate::bft::message::{ConsensusMessageKind, PBFTMessage};
 
 pub type StoredConsensusMessage<O> = ShareableMessage<PBFTMessage<O>>;
 
@@ -31,7 +31,7 @@ pub struct ProofMetadata {
     seq_no: SeqNo,
     batch_digest: Digest,
     pre_prepare_ordering: Vec<Digest>,
-    contained_client_rqs: usize
+    contained_client_rqs: usize,
 }
 
 impl Orderable for ProofMetadata {
@@ -90,7 +90,7 @@ impl ProofMetadata {
             seq_no,
             batch_digest: digest,
             pre_prepare_ordering,
-            contained_client_rqs: contained_rqs
+            contained_client_rqs: contained_rqs,
         }
     }
 
@@ -124,6 +124,33 @@ impl<O> Proof<O> {
         }
     }
 
+    pub fn init_from_messages(metadata: ProofMetadata, messages: Vec<StoredConsensusMessage<O>>) -> Self {
+        let mut pre_prepares = Vec::new();
+        let mut prepares = Vec::new();
+        let mut commits = Vec::new();
+
+        for x in messages {
+            match x.message().consensus().kind() {
+                ConsensusMessageKind::PrePrepare(_) => {
+                    pre_prepares.push(x);
+                }
+                ConsensusMessageKind::Prepare(_) => {
+                    prepares.push(x);
+                }
+                ConsensusMessageKind::Commit(_) => {
+                    commits.push(x);
+                }
+            }
+        }
+
+        Self {
+            metadata,
+            pre_prepares,
+            prepares,
+            commits,
+        }
+    }
+
     pub(crate) fn metadata(&self) -> &ProofMetadata {
         &self.metadata
     }
@@ -199,7 +226,6 @@ impl<O> Proof<O> {
     }
 
     pub fn into_parts(self) -> (ProofMetadata, Vec<ShareableMessage<PBFTMessage<O>>>) {
-
         let mut vec = Vec::with_capacity(self.pre_prepares.len() + self.prepares.len() + self.commits.len());
 
         for pre_prepares in self.pre_prepares {
@@ -216,7 +242,6 @@ impl<O> Proof<O> {
 
         (self.metadata, vec)
     }
-
 }
 
 impl<O> Orderable for Proof<O> {
@@ -273,7 +298,6 @@ pub struct CollectData<O> {
 }
 
 impl<O> CollectData<O> {
-
     pub fn new(incomplete_proof: IncompleteProof, last_proof: Option<Proof<O>>) -> Self {
         Self { incomplete_proof, last_proof }
     }
diff --git a/febft-pbft-consensus/src/bft/log/mod.rs b/febft-pbft-consensus/src/bft/log/mod.rs
index c87b59f6..154ac2ca 100644
--- a/febft-pbft-consensus/src/bft/log/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/mod.rs
@@ -50,10 +50,11 @@ impl<D> Log<D> where D: ApplicationData {
         }
 
         let batch_info = ProtocolConsensusDecision::from(&proof);
+        let sequence = proof.sequence_number();
 
         let (metadata, messages) = proof.into_parts();
 
-        Ok(Decision::full_decision_info(proof.sequence_number(), metadata, messages, batch_info))
+        Ok(Decision::full_decision_info(sequence, metadata, messages, batch_info))
     }
 
     pub fn finalize_batch(&mut self, completed: CompletedBatch<D::Request>) -> Result<ProtocolConsensusDecision<D::Request>> {
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index 4bbd99f0..5a352bd4 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -6,7 +6,6 @@
 //! as [Cap'n'Proto](https://capnproto.org/capnp-tool.html), but these are
 //! expected to be implemented by the user.
 
-use std::fmt::Debug;
 use std::io::{Read, Write};
 use std::marker::PhantomData;
 use std::sync::Arc;
@@ -16,17 +15,17 @@ use ::serde::{Deserialize, Serialize};
 use bytes::Bytes;
 
 use atlas_common::error::*;
-use atlas_communication::message::Header;
-use atlas_communication::message_signing::NetworkMessageSignatureVerifier;
+use atlas_common::ordering::Orderable;
+use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
-use atlas_communication::serialize::Serializable;
 use atlas_core::ordering_protocol::loggable::PersistentOrderProtocolTypes;
 use atlas_core::ordering_protocol::networking::serialize::{OrderingProtocolMessage, PermissionedOrderingProtocolMessage};
 use atlas_core::ordering_protocol::networking::signature_ver::OrderProtocolSignatureVerificationHelper;
 use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::log::decisions::{Proof, ProofMetadata};
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
+use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, FwdConsensusMessage, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
+use crate::bft::sync::LeaderCollects;
 use crate::bft::sync::view::ViewInfo;
 
 #[cfg(feature = "serialize_capnp")]
@@ -69,143 +68,110 @@ pub fn deserialize_consensus<R, D>(r: R) -> Result<ConsensusMessage<D::Request>>
 /// The serializable type, to be used to appease the compiler and it's requirements
 pub struct PBFTConsensus<D: ApplicationData>(PhantomData<(D)>);
 
-impl<D> PBFTConsensus<D> where D: ApplicationData {
-    /// Verify the consensus internal message structure
-    fn verify_consensus_message<S, SV, NI>(network_info: &Arc<NI>, header: &Header, msg: &ConsensusMessage<D::Request>) -> Result<bool>
-        where S: Serializable,
-              NI: NetworkInformationProvider,
-              SV: NetworkMessageSignatureVerifier<S, NI>, {
-        match msg.kind() {
-            ConsensusMessageKind::PrePrepare(requests) => {
-                requests.iter().map(|request| {
-                    let header = request.header();
-                    let client_request = request.message();
-
-                    //TODO: Verify client request signature
-
-                    Ok(false)
-                }).reduce(|a, b| a.and(b))
-                    .unwrap_or(Ok(true))
-            }
-            ConsensusMessageKind::Prepare(prepare) => {
-                Ok(true)
-            }
-            ConsensusMessageKind::Commit(commit) => {
-                Ok(true)
-            }
-        }
-    }
-
-    /// Verify view change message internal structure
-    fn verify_view_change_message<S, SV, NI>(network_info: &Arc<NI>, header: &Header, msg: &ViewChangeMessage<D::Request>) -> Result<bool>
-        where S: Serializable,
-              NI: NetworkInformationProvider,
-              SV: NetworkMessageSignatureVerifier<S, NI>, {
-        match msg.kind() {
-            ViewChangeMessageKind::Stop(timed_out_rqs) => {
-                timed_out_rqs.iter().map(|request| {
-                    let header = request.header();
-                    let client_request = request.message();
-
-                    Ok(false)
-                }).reduce(|a, b| a.and(b)).unwrap_or(Ok(true))
-            }
-            ViewChangeMessageKind::StopQuorumJoin(joining_node) => {
-                Ok(true)
-            }
-            ViewChangeMessageKind::StopData(data) => {
-                if let Some(proof) = &data.last_proof {
-                    proof.pre_prepares().iter().map(|pre_prepare| {
-                        let header = pre_prepare.header();
-                        let consensus_message = pre_prepare.message();
-
-                        Self::verify_consensus_message::<S, SV, NI>(network_info, header, consensus_message.consensus())
-                    }).reduce(|a, b| a.and(b)).unwrap_or(Ok(true))
-                } else {
-                    Ok(true)
-                }
-            }
-            ViewChangeMessageKind::Sync(sync_message) => {
-                let message = sync_message.message();
-
-                let header = message.header();
-                let message = message.consensus();
-
-                Self::verify_consensus_message::<S, SV, NI>(network_info, header, message)
-            }
-        }
-    }
-}
-
 impl<D> OrderingProtocolMessage<D> for PBFTConsensus<D>
     where D: ApplicationData, {
     type ProtocolMessage = PBFTMessage<D::Request>;
     type ProofMetadata = ProofMetadata;
 
-    fn verify_order_protocol_message<NI, OPVH>(network_info: &Arc<NI>, header: &Header, message: Self::ProtocolMessage) -> Result<(bool, Self::ProtocolMessage)> where NI: NetworkInformationProvider, OPVH: OrderProtocolSignatureVerificationHelper<D, Self, NI>, Self: Sized {
-        match &message {
+    fn verify_order_protocol_message<NI, OPVH>(network_info: &Arc<NI>, header: &Header, message: Self::ProtocolMessage) -> Result<Self::ProtocolMessage>
+        where NI: NetworkInformationProvider,
+              OPVH: OrderProtocolSignatureVerificationHelper<D, Self, NI>, Self: Sized {
+        match message {
             PBFTMessage::Consensus(consensus) => {
-                match consensus.kind() {
+                let (seq, view) = (consensus.sequence_number(), consensus.view());
+
+                match consensus.into_kind() {
                     ConsensusMessageKind::PrePrepare(requests) => {
-                        for request in requests {
-                            let (result, _) = OPVH::verify_request_message(network_info, request.header(), request.message().clone())?;
+                        let mut request_copy = Vec::with_capacity(requests.len());
+
+                        let request_iter = requests.into_iter();
+
+                        let mut global_res = true;
+
+                        for request in request_iter {
+                            let (header, message) = request.into_inner();
+
+                            let message = OPVH::verify_request_message(network_info, &header, message)?;
 
-                            if !result {
-                                return Ok((false, message));
-                            }
+                            let stored_msg = StoredMessage::new(header, message);
+
+                            request_copy.push(stored_msg);
                         }
 
-                        Ok((true, message))
+                        let consensus = ConsensusMessage::new(seq, view, ConsensusMessageKind::PrePrepare(request_copy));
+
+                        Ok(PBFTMessage::Consensus(consensus))
                     }
                     ConsensusMessageKind::Prepare(digest) => {
-                        Ok((true, message))
+                        Ok(PBFTMessage::Consensus(ConsensusMessage::new(seq, view, ConsensusMessageKind::Prepare(digest))))
                     }
                     ConsensusMessageKind::Commit(digest) => {
-                        Ok((true, message))
+                        Ok(PBFTMessage::Consensus(ConsensusMessage::new(seq, view, ConsensusMessageKind::Commit(digest))))
                     }
                 }
             }
             PBFTMessage::ViewChange(view_change) => {
-                match view_change.kind() {
+                let (view) = view_change.sequence_number();
+
+                match view_change.into_kind() {
                     ViewChangeMessageKind::Stop(timed_out_req) => {
-                        for client_rq in timed_out_req {
-                            let (result, _) = OPVH::verify_request_message(network_info, client_rq.header(), client_rq.message().clone())?;
+                        let mut rq_copy = Vec::with_capacity(timed_out_req.len());
+
+                        let rq_iter = timed_out_req.into_iter();
+
+                        for client_rq in rq_iter {
+                            let (header, message) = client_rq.into_inner();
+
+                            let rq_message = OPVH::verify_request_message(network_info, &header, message)?;
 
-                            if !result {
-                                return Ok((false, message));
-                            }
+                            let stored_rq = StoredMessage::new(header, rq_message);
+
+                            rq_copy.push(stored_rq);
                         }
 
-                        Ok((true, message))
+                        Ok(PBFTMessage::ViewChange(ViewChangeMessage::new(view, ViewChangeMessageKind::Stop(rq_copy))))
                     }
-                    ViewChangeMessageKind::StopQuorumJoin(_) => {
-                        Ok((true, message))
+                    ViewChangeMessageKind::StopQuorumJoin(node) => {
+                        Ok(PBFTMessage::ViewChange(ViewChangeMessage::new(view, ViewChangeMessageKind::StopQuorumJoin(node))))
                     }
                     ViewChangeMessageKind::StopData(collect_data) => {
                         if let Some(proof) = &collect_data.last_proof {}
 
-                        Ok((true, message))
+                        let vcm = ViewChangeMessage::new(view, ViewChangeMessageKind::StopData(collect_data));
+                        Ok(PBFTMessage::ViewChange(vcm))
                     }
                     ViewChangeMessageKind::Sync(leader_collects) => {
-                        let (result, _) = OPVH::verify_protocol_message(network_info, leader_collects.message().header(), PBFTMessage::Consensus(leader_collects.message().consensus().clone()))?;
+                        let (fwd, collects) = leader_collects.into_inner();
 
-                        if !result {
-                            return Ok((false, message));
-                        }
+                        let res = {
+                            let (header, message) = fwd.into_inner();
+
+                            let message = OPVH::verify_protocol_message(network_info, &header, PBFTMessage::Consensus(message))?;
+
+                            let message = FwdConsensusMessage::new(header, message.into_consensus());
+
+                            message
+                        };
 
-                        for collect in leader_collects.collects() {
-                            let (result, _) = OPVH::verify_protocol_message(network_info, collect.header(), PBFTMessage::ViewChange(collect.message().clone()))?;
+                        let mut collected_messages = Vec::with_capacity(collects.len());
 
-                            if !result {
-                                return Ok((false, message));
-                            }
+                        let iter = collects.into_iter();
+
+                        for collect in iter {
+                            let (header, message) = collect.into_inner();
+
+                            let message = OPVH::verify_protocol_message(network_info, &header, message)?;
+
+                            collected_messages.push(StoredMessage::new(header, message));
                         }
 
-                        Ok((true, message))
+                        let vc = ViewChangeMessage::new(view, ViewChangeMessageKind::Sync(LeaderCollects::new(res, collected_messages)));
+
+                        Ok(PBFTMessage::ViewChange(vc))
                     }
                 }
             }
-            PBFTMessage::ObserverMessage(_) => Ok((true, message))
+            PBFTMessage::ObserverMessage(m) => Ok(PBFTMessage::ObserverMessage(m))
         }
     }
 
@@ -248,34 +214,19 @@ impl<D> PersistentOrderProtocolTypes<D, Self> for PBFTConsensus<D>
     where D: ApplicationData + 'static {
     type Proof = Proof<D::Request>;
 
-    fn verify_proof<NI, OPVH>(network_info: &Arc<NI>, proof: Self::Proof) -> Result<(bool, Self::Proof)>
+    fn verify_proof<NI, OPVH>(network_info: &Arc<NI>, proof: Self::Proof) -> Result<Self::Proof>
         where NI: NetworkInformationProvider,
               OPVH: OrderProtocolSignatureVerificationHelper<D, Self, NI>,
               Self: Sized {
-        for pre_prepare in proof.pre_prepares() {
-            let (result, _) = OPVH::verify_protocol_message(network_info, pre_prepare.header(), pre_prepare.message().clone())?;
 
-            if !result {
-                return Ok((false, proof));
-            }
-        }
+        let (metadata, messages) = proof.into_parts();
 
-        for prepare in proof.prepares() {
-            let (result, _) = OPVH::verify_protocol_message(network_info, prepare.header(), prepare.message().clone())?;
-
-            if !result {
-                return Ok((false, proof));
-            }
+        for msg in &messages {
+            let _ = OPVH::verify_protocol_message(network_info, msg.header(), msg.message().clone())?;
         }
 
-        for commit in proof.commits() {
-            let (result, _) = OPVH::verify_protocol_message(network_info, commit.header(), commit.message().clone())?;
-
-            if !result {
-                return Ok((false, proof));
-            }
-        }
+        let proof = Proof::init_from_messages(metadata, messages);
 
-        return Ok((true, proof));
+        Ok(proof)
     }
 }
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index 505e78b6..ae9109e4 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -18,8 +18,9 @@ use serde::{Deserialize, Serialize};
 use atlas_common::{collections, prng};
 use atlas_common::crypto::hash::Digest;
 use atlas_common::error::*;
+use atlas_common::globals::ReadOnly;
 use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_pop_message, tbo_queue_message};
+use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_pop_message, tbo_queue_message, tbo_queue_message_arc};
 use atlas_communication::message::{Header, StoredMessage, WireMessage};
 use atlas_communication::reconfiguration_node::NetworkInformationProvider;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
@@ -28,7 +29,7 @@ use atlas_core::ordering_protocol::{Decision, ProtocolConsensusDecision};
 use atlas_core::ordering_protocol::reconfigurable_order_protocol::ReconfigurationAttemptResult;
 use atlas_core::persistent_log::{OrderingProtocolLog};
 use atlas_core::request_pre_processing::RequestPreProcessor;
-use atlas_core::smr::smr_decision_log::ShareableMessage;
+use atlas_core::smr::smr_decision_log::{ShareableMessage, unwrap_shareable_message};
 use atlas_core::timeouts::{RqTimeout, Timeouts};
 use atlas_smr_application::serialize::ApplicationData;
 
@@ -117,15 +118,23 @@ pub struct LeaderCollects<O> {
     // Done
     proposed: FwdConsensusMessage<O>,
     // The collect messages the leader has received.
-    collects: Vec<ShareableMessage<PBFTMessage<O>>>,
+    collects: Vec<StoredMessage<PBFTMessage<O>>>,
 }
 
 impl<O> LeaderCollects<O> {
+    pub fn new(proposed: FwdConsensusMessage<O>,
+               collects: Vec<StoredMessage<PBFTMessage<O>>>) -> Self {
+        Self {
+            proposed,
+            collects,
+        }
+    }
+
     pub fn message(&self) -> &FwdConsensusMessage<O> {
         &self.proposed
     }
 
-    pub fn collects(&self) -> &Vec<ShareableMessage<PBFTMessage<O>>> {
+    pub fn collects(&self) -> &Vec<StoredMessage<PBFTMessage<O>>> {
         &self.collects
     }
 
@@ -134,7 +143,7 @@ impl<O> LeaderCollects<O> {
         self,
     ) -> (
         FwdConsensusMessage<O>,
-        Vec<ShareableMessage<PBFTMessage<O>>>,
+        Vec<StoredMessage<PBFTMessage<O>>>,
     ) {
         (self.proposed, self.collects)
     }
@@ -301,21 +310,21 @@ impl<O> TboQueue<O> {
         // NOTE: we use next() because we want to retrieve messages
         // for v+1, as we haven't started installing the new view yet
         let seq = self.view.sequence_number().next();
-        tbo_queue_message(seq, &mut self.stop, m)
+        tbo_queue_message_arc(seq, &mut self.stop, (m.sequence_number(), m))
     }
 
     /// Queues a `STOP-DATA` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
     fn queue_stop_data(&mut self, m: ShareableMessage<PBFTMessage<O>>) {
         let seq = self.view.sequence_number().next();
-        tbo_queue_message(seq, &mut self.stop_data, m)
+        tbo_queue_message_arc(seq, &mut self.stop_data, (m.sequence_number(), m))
     }
 
     /// Queues a `SYNC` message for later processing, or drops it
     /// immediately if it pertains to an older view change instance.
     fn queue_sync(&mut self, m: ShareableMessage<PBFTMessage<O>>) {
         let seq = self.view.sequence_number().next();
-        tbo_queue_message(seq, &mut self.sync, m)
+        tbo_queue_message_arc(seq, &mut self.sync, (m.sequence_number(), m))
     }
 
     pub fn view(&self) -> &ViewInfo {
@@ -429,7 +438,7 @@ pub trait AbstractSynchronizer<D> where D: ApplicationData + 'static {
     fn queue(&self, message: ShareableMessage<PBFTMessage<D::Request>>);
 }
 
-type CollectsType<D: ApplicationData> = IntMap<ShareableMessage<PBFTMessage<D::Request>>>;
+type CollectsType<D: ApplicationData> = IntMap<StoredMessage<PBFTMessage<D::Request>>>;
 
 ///The synchronizer for the SMR protocol
 /// This part of the protocol is responsible for handling the changing of views and
@@ -691,16 +700,16 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
         where NT: OrderProtocolSendNode<D, PBFT<D>> + 'static,
     {
 
-        let (header, message) = (s_message.header(), s_message.message().view_change());
-
         debug!("{:?} // Processing view change message {:?} in phase {:?} from {:?}",
                node.id(),
-               message,
+               s_message.message().view_change(),
                self.phase.get(),
-               header.from());
+               s_message.header().from());
 
         match self.phase.get() {
             ProtoPhase::Init => {
+                let (header, message) = (s_message.header(), s_message.message().view_change());
+
                 return match message.kind() {
                     ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_) => {
                         let mut guard = self.tbo.lock().unwrap();
@@ -739,6 +748,8 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                 };
             }
             ProtoPhase::Stopping(i) | ProtoPhase::Stopping2(i) => {
+                let (header, message) = (s_message.header(), s_message.message().view_change());
+
                 let msg_seq = message.sequence_number();
                 let current_view = self.view();
                 let next_seq = current_view.sequence_number().next();
@@ -798,8 +809,8 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                 };
 
                 // store pending requests from this STOP
-                let mut stopped = match message.into_kind() {
-                    ViewChangeMessageKind::Stop(stopped) => stopped,
+                let mut stopped = match message.kind() {
+                    ViewChangeMessageKind::Stop(stopped) => stopped.clone(),
                     _ => unreachable!(),
                 };
 
@@ -858,6 +869,8 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                 SynchronizerStatus::Running
             }
             ProtoPhase::ViewStopping(received) | ProtoPhase::ViewStopping2(received) => {
+                let (header, message) = (s_message.header(), s_message.message().view_change());
+
                 let msg_seq = message.sequence_number();
                 let current_view = self.view();
                 let next_seq = current_view.sequence_number().next();
@@ -988,6 +1001,8 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                 SynchronizerStatus::Running
             }
             ProtoPhase::StoppingData(i) => {
+                let (header, message) = (s_message.header(), s_message.message().view_change());
+
                 match &self.accessory {
                     SynchronizerAccessory::Follower(_) => {
                         //Since a follower can never be a leader (as he isn't a part of the
@@ -1078,7 +1093,11 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                         // the new leader isn't forging messages.
 
                         // store collects from this STOP-DATA
-                        collects_guard.insert(header.from().into(), s_message);
+                        let unwrapped_msg = unwrap_shareable_message(s_message);
+
+                        let (header, message) = (unwrapped_msg.header(), unwrapped_msg.message());
+
+                        collects_guard.insert(header.from().into(), unwrapped_msg);
 
                         if i < next_view.params().quorum() {
                             self.phase.replace(ProtoPhase::StoppingData(i));
@@ -1209,12 +1228,12 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                 }
             }
             ProtoPhase::Syncing => {
-                let msg_seq = message.sequence_number();
+                let msg_seq = s_message.sequence_number();
                 let next_view = self.next_view().expect("We should have a next view in this situation");
                 let seq = next_view.sequence_number();
 
                 // reject SYNC messages if these were not sent by the leader
-                let (proposed, collects) = match message.kind() {
+                let (proposed, collects) = match s_message.message().view_change().kind() {
                     ViewChangeMessageKind::Stop(_) | ViewChangeMessageKind::StopQuorumJoin(_) => {
                         {
                             let mut guard = self.tbo.lock().unwrap();
@@ -1250,7 +1269,8 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
                     }
                     ViewChangeMessageKind::Sync(_) if msg_seq != seq => {
                         {
-                            debug!("{:?} // Received sync message whose sequence number does not match our current one {:?} vs {:?}. Queueing", node.id(), message, next_view);
+                            debug!("{:?} // Received sync message whose sequence number does not match our current one {:?} vs {:?}. Queueing", node.id(),
+                                s_message.message().view_change(), next_view);
 
                             let mut guard = self.tbo.lock().unwrap();
 
@@ -1259,14 +1279,16 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
                         return SynchronizerStatus::Running;
                     }
-                    ViewChangeMessageKind::Sync(_) if header.from() != next_view.leader() => {
+                    ViewChangeMessageKind::Sync(_) if s_message.header().from() != next_view.leader() => {
                         //You're not the leader, what are you saying
                         return SynchronizerStatus::Running;
                     }
                     ViewChangeMessageKind::Sync(_) => {
-                        let message = message;
+                        let stored_message = unwrap_shareable_message(s_message);
+
+                        let (header, message) = stored_message.into_inner();
 
-                        message.take_collects().unwrap().into_inner()
+                        message.into_view_change().take_collects().unwrap().into_inner()
                     }
                 };
 
@@ -1443,10 +1465,10 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 
     /// Trigger a view change locally
     pub fn begin_quorum_view_change<NT>(&self,
-                                            join_cert: Option<NodeId>,
-                                            node: &NT,
-                                            timeouts: &Timeouts,
-                                            _log: &Log<D>, )
+                                        join_cert: Option<NodeId>,
+                                        node: &NT,
+                                        timeouts: &Timeouts,
+                                        _log: &Log<D>, )
         where NT: OrderProtocolSendNode<D, PBFT<D>>,
     {
         debug!("Beginning quorum view change with certificate {} at phase {:?}",  join_cert.is_some(), self.phase.get());
@@ -1732,7 +1754,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
 // may be executing the same CID when there is a leader change
     #[inline]
     fn normalized_collects<'a>(
-        collects: &'a IntMap<ShareableMessage<PBFTMessage<D::Request>>>,
+        collects: &'a IntMap<StoredMessage<PBFTMessage<D::Request>>>,
         in_exec: SeqNo,
     ) -> impl Iterator<Item=Option<&'a CollectData<D::Request>>> {
         let values = collects.values();
@@ -1745,7 +1767,7 @@ impl<D> Synchronizer<D> where D: ApplicationData + 'static,
     // TODO: quorum sizes may differ when we implement reconfiguration
     #[inline]
     fn highest_proof<'a, NT>(
-        guard: &'a IntMap<ShareableMessage<PBFTMessage<D::Request>>>,
+        guard: &'a IntMap<StoredMessage<PBFTMessage<D::Request>>>,
         view: &ViewInfo,
         node: &NT,
     ) -> Option<&'a Proof<D::Request>>
@@ -1955,7 +1977,7 @@ fn certified_value<O>(
 }
 
 fn collect_data<'a, O: 'a>(
-    collects: impl Iterator<Item=&'a ShareableMessage<PBFTMessage<O>>>,
+    collects: impl Iterator<Item=&'a StoredMessage<PBFTMessage<O>>>,
 ) -> impl Iterator<Item=&'a CollectData<O>> {
     collects.filter_map(|stored| match stored.message().view_change().kind() {
         ViewChangeMessageKind::StopData(collects) => Some(collects),
@@ -1978,14 +2000,14 @@ fn normalized_collects<'a, O: 'a>(
 
 fn signed_collects<D, NT>(
     node: &NT,
-    collects: Vec<ShareableMessage<PBFTMessage<D::Request>>>,
-) -> Vec<ShareableMessage<PBFTMessage<D::Request>>>
+    collects: Vec<StoredMessage<PBFTMessage<D::Request>>>,
+) -> Vec<StoredMessage<PBFTMessage<D::Request>>>
     where D: ApplicationData + 'static,
           NT: OrderProtocolSendNode<D, PBFT<D>>
 {
     collects
         .into_iter()
-        .filter(|stored| validate_signature::<D, _, _>(node, &**stored))
+        .filter(|stored| validate_signature::<D, _, _>(node, &stored))
         .collect()
 }
 
@@ -2025,7 +2047,7 @@ fn highest_proof<'a, D, I, NT>(
 ) -> Option<&'a Proof<D::Request>>
     where
         D: ApplicationData + 'static,
-        I: Iterator<Item=&'a ShareableMessage<PBFTMessage<D::Request>>>,
+        I: Iterator<Item=&'a StoredMessage<PBFTMessage<D::Request>>>,
         NT: OrderProtocolSendNode<D, PBFT<D>>
 {
     collect_data(collects)
diff --git a/febft-state-transfer/src/message/serialize/mod.rs b/febft-state-transfer/src/message/serialize/mod.rs
index 265f6be1..c0fc0dd5 100644
--- a/febft-state-transfer/src/message/serialize/mod.rs
+++ b/febft-state-transfer/src/message/serialize/mod.rs
@@ -17,7 +17,7 @@ pub struct CSTMsg<S: MonolithicState>(PhantomData<(S)>);
 impl<S: MonolithicState> StateTransferMessage for CSTMsg<S> {
     type StateTransferMessage = CstMessage<S>;
 
-    fn verify_state_message<NI, SVH>(network_info: &Arc<NI>, header: &Header, message: Self::StateTransferMessage) -> atlas_common::error::Result<(bool, Self::StateTransferMessage)>
+    fn verify_state_message<NI, SVH>(network_info: &Arc<NI>, header: &Header, message: Self::StateTransferMessage) -> atlas_common::error::Result<Self::StateTransferMessage>
         where NI: NetworkInformationProvider, SVH: StateTransferVerificationHelper {
         todo!()
     }

From 7671b29495c4300cd19305afffa37fddcb40d1e9 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 18 Oct 2023 23:56:31 +0100
Subject: [PATCH 65/80] Well I was still not counting the decision log, some
 more compilation errors to fix.

---
 febft-pbft-consensus/src/bft/log/deciding/mod.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
index 5c22eb09..b023aefb 100644
--- a/febft-pbft-consensus/src/bft/log/deciding/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -321,8 +321,7 @@ impl<O> CompletedBatch<O> {
     pub fn request_count(&self) -> usize {
         self.client_requests.len()
     }
-
-
+    
 }
 
 impl<O> Orderable for WorkingDecisionLog<O> {

From 970bd79ad961a380ac305a6fbfecdf1aabed00a1 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 19 Oct 2023 17:46:24 +0100
Subject: [PATCH 66/80] Fixed some execution issues. Fixed infinite loop in
 decision log. Fixed lack of logging in order protocol, leading to bad
 unwraps.

---
 .../src/bft/consensus/decision/mod.rs         | 14 +++----
 febft-pbft-consensus/src/bft/consensus/mod.rs |  8 +++-
 .../src/bft/log/deciding/mod.rs               | 13 ++++--
 .../src/bft/log/decisions/mod.rs              | 23 +++++++---
 .../src/bft/message/serialize/mod.rs          |  2 +-
 febft-pbft-consensus/src/bft/mod.rs           | 42 +++----------------
 6 files changed, 47 insertions(+), 55 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
index ddc21502..d149b928 100644
--- a/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/decision/mod.rs
@@ -20,7 +20,7 @@ use atlas_smr_application::serialize::ApplicationData;
 use crate::bft::consensus::accessory::{AccessoryConsensus, ConsensusDecisionAccessory};
 use crate::bft::consensus::accessory::replica::ReplicaAccessory;
 use crate::bft::log::deciding::{CompletedBatch, WorkingDecisionLog};
-use crate::bft::log::decisions::IncompleteProof;
+use crate::bft::log::decisions::{IncompleteProof, ProofMetadata};
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::metric::{ConsensusMetrics, PRE_PREPARE_ANALYSIS_ID};
 use crate::bft::PBFT;
@@ -82,7 +82,7 @@ pub enum DecisionStatus<O> {
     /// on a client request to be executed.
     Deciding(ShareableMessage<PBFTMessage<O>>),
     /// Transitioned to another next phase of the consensus decision
-    Transitioned(ShareableMessage<PBFTMessage<O>>),
+    Transitioned(Option<ProofMetadata>, ShareableMessage<PBFTMessage<O>>),
     /// A `febft` quorum decided on the execution of
     /// the batch of requests with the given digests.
     /// The first digest is the digest of the Prepare message
@@ -335,7 +335,7 @@ impl<D> ConsensusDecision<D>
                     &mut self.working_log,
                 );
 
-                let batch_metadata = self.working_log.process_pre_prepare(header.clone(), message,
+                let batch_metadata = self.working_log.process_pre_prepare(s_message.clone(),
                                                                           header.digest().clone(), digests)?;
 
                 let mut result;
@@ -366,7 +366,7 @@ impl<D> ConsensusDecision<D>
                     self.message_queue.signal();
 
                     // Mark that we have transitioned to the next phase
-                    result = DecisionStatus::Transitioned(s_message);
+                    result = DecisionStatus::Transitioned(Some(batch_metadata), s_message);
 
                     // We no longer start the count at 1 since all leaders must also send the prepare
                     // message with the digest of the entire batch
@@ -433,7 +433,7 @@ impl<D> ConsensusDecision<D>
                     self.consensus_metrics.first_prepare_recvd();
                 }
 
-                self.working_log.process_message(&header, &message)?;
+                self.working_log.process_message(s_message.clone())?;
 
                 let result;
 
@@ -451,7 +451,7 @@ impl<D> ConsensusDecision<D>
 
                     self.message_queue.signal();
 
-                    result = DecisionStatus::Transitioned(s_message);
+                    result = DecisionStatus::Transitioned(None, s_message);
 
                     DecisionPhase::Committing(0)
                 } else {
@@ -505,7 +505,7 @@ impl<D> ConsensusDecision<D>
                     self.consensus_metrics.first_commit_recvd();
                 }
 
-                self.working_log.process_message(&header, &message);
+                self.working_log.process_message(s_message.clone())?;
 
                 return if received == view.params().quorum() {
                     info!("{:?} // Completed commit phase with all commits Seq {:?} with commit from {:?}", node.id(), self.sequence_number(),
diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 4522c6ed..1815c662 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -428,12 +428,16 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
 
                 ConsensusStatus::MessageQueued
             }
-            DecisionStatus::Transitioned(message) => {
+            DecisionStatus::Transitioned(metadata, message) => {
                 //When we transition phases, we may discover new messages
                 // That were in the queue, so we must be signalled again
                 self.signalled.push_signalled(decision_seq);
 
-                ConsensusStatus::Deciding(MaybeVec::from_one(Decision::decision_info_from_message(decision_seq, message)))
+                if let Some(metadata) = metadata {
+                    ConsensusStatus::Deciding(MaybeVec::from_one(Decision::decision_info_from_metadata_and_messages(decision_seq, metadata, MaybeVec::from_one(message))))
+                } else {
+                    ConsensusStatus::Deciding(MaybeVec::from_one(Decision::decision_info_from_message(decision_seq, message)))
+                }
             }
             DecisionStatus::Decided(message) => {
                 ConsensusStatus::Decided(MaybeVec::from_one(Decision::decision_info_from_message(decision_seq, message)))
diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
index b023aefb..2cf519fa 100644
--- a/febft-pbft-consensus/src/bft/log/deciding/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -150,10 +150,11 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
     }
 
     pub fn process_pre_prepare(&mut self,
-                               header: Header,
-                               message: &ConsensusMessage<O>,
+                               s_message: ShareableMessage<PBFTMessage<O>>,
                                digest: Digest,
                                mut batch_rq_digests: Vec<ClientRqInfo>) -> Result<Option<ProofMetadata>> {
+        let (header, message) = (s_message.header(), s_message.message().consensus());
+
         let start = Instant::now();
 
         let sending_leader = header.from();
@@ -198,6 +199,8 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
 
         self.client_rqs.append(&mut batch_rq_digests);
 
+        self.message_log.insert_pre_prepare(leader_index, s_message);
+
         metric_duration(PRE_PREPARE_LOG_ANALYSIS_ID, start.elapsed());
 
         // if we have received all of the messages in the set, calculate the digest.
@@ -217,13 +220,17 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
     }
 
     /// Process the message received
-    pub(crate) fn process_message(&mut self, header: &Header, message: &ConsensusMessage<O>) -> Result<()> {
+    pub(crate) fn process_message(&mut self, s_message: ShareableMessage<PBFTMessage<O>>) -> Result<()> {
+        let (header, message) = (s_message.header(), s_message.message().consensus());
+
         match message.kind() {
             ConsensusMessageKind::Prepare(_) => {
                 self.duplicate_detection.insert_prepare_received(header.from())?;
+                self.message_log.insert_prepare(s_message);
             }
             ConsensusMessageKind::Commit(_) => {
                 self.duplicate_detection.insert_commit_received(header.from())?;
+                self.message_log.insert_commit(s_message);
             }
             _ => unreachable!()
         }
diff --git a/febft-pbft-consensus/src/bft/log/decisions/mod.rs b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
index 1fedd0ea..16ba2ff6 100644
--- a/febft-pbft-consensus/src/bft/log/decisions/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/decisions/mod.rs
@@ -1,4 +1,5 @@
 use std::fmt::{Debug, Formatter};
+use std::iter;
 use std::ops::Deref;
 
 #[cfg(feature = "serialize_serde")]
@@ -124,15 +125,19 @@ impl<O> Proof<O> {
         }
     }
 
-    pub fn init_from_messages(metadata: ProofMetadata, messages: Vec<StoredConsensusMessage<O>>) -> Self {
-        let mut pre_prepares = Vec::new();
+    pub fn init_from_messages(metadata: ProofMetadata, messages: Vec<StoredConsensusMessage<O>>) -> Result<Self> {
+        let mut pre_prepares: Vec<Option<StoredConsensusMessage<O>>> = iter::repeat(None).take(metadata.pre_prepare_ordering().len()).collect();
         let mut prepares = Vec::new();
         let mut commits = Vec::new();
 
         for x in messages {
             match x.message().consensus().kind() {
                 ConsensusMessageKind::PrePrepare(_) => {
-                    pre_prepares.push(x);
+                    let option = metadata.pre_prepare_ordering().iter().position(|digest| *x.header().digest() == *digest);
+
+                    let index = option.ok_or(Error::simple_with_msg(ErrorKind::OrderProtocolProof, "Failed to create proof as pre prepare is not contained in metadata ordering"))?;
+
+                    pre_prepares[index] = Some(x);
                 }
                 ConsensusMessageKind::Prepare(_) => {
                     prepares.push(x);
@@ -143,12 +148,18 @@ impl<O> Proof<O> {
             }
         }
 
-        Self {
+        let mut pre_prepares_f = Vec::with_capacity(metadata.pre_prepare_ordering().len());
+
+        for message in pre_prepares.into_iter() {
+            pre_prepares_f.push(message.ok_or(Error::simple_with_msg(ErrorKind::OrderProtocolProof, "Failed to create proof as pre prepare list is not complete"))?);
+        }
+
+        Ok(Self {
             metadata,
-            pre_prepares,
+            pre_prepares: pre_prepares_f,
             prepares,
             commits,
-        }
+        })
     }
 
     pub(crate) fn metadata(&self) -> &ProofMetadata {
diff --git a/febft-pbft-consensus/src/bft/message/serialize/mod.rs b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
index 5a352bd4..a36b2f56 100644
--- a/febft-pbft-consensus/src/bft/message/serialize/mod.rs
+++ b/febft-pbft-consensus/src/bft/message/serialize/mod.rs
@@ -225,7 +225,7 @@ impl<D> PersistentOrderProtocolTypes<D, Self> for PBFTConsensus<D>
             let _ = OPVH::verify_protocol_message(network_info, msg.header(), msg.message().clone())?;
         }
 
-        let proof = Proof::init_from_messages(metadata, messages);
+        let proof = Proof::init_from_messages(metadata, messages)?;
 
         Ok(proof)
     }
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 74ab0901..fd7911ae 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -803,49 +803,19 @@ impl<D, NT> OrderProtocolPersistenceHelper<D, PBFTConsensus<D>, PBFTConsensus<D>
         }
     }
 
-    fn init_proof_from(metadata: ProofMetadata, messages: Vec<StoredMessage<PBFTMessage<D::Request>>>) -> Proof<D::Request> {
-        let mut pre_prepares = Vec::with_capacity(messages.len() / 2);
-        let mut prepares = Vec::with_capacity(messages.len() / 2);
-        let mut commits = Vec::with_capacity(messages.len() / 2);
+    fn init_proof_from(metadata: ProofMetadata, messages: Vec<StoredMessage<PBFTMessage<D::Request>>>) -> Result<Proof<D::Request>> {
+        let mut messages_f = Vec::with_capacity(messages.len());
 
         for message in messages {
-            match message.message().consensus().kind() {
-                ConsensusMessageKind::PrePrepare(_) => {
-                    pre_prepares.push(Arc::new(ReadOnly::new(message)));
-                }
-                ConsensusMessageKind::Prepare(_) => {
-                    prepares.push(Arc::new(ReadOnly::new(message)));
-                }
-                ConsensusMessageKind::Commit(_) => {
-                    commits.push(Arc::new(ReadOnly::new(message)));
-                }
-            }
+            messages_f.push(Arc::new(ReadOnly::new(message)));
         }
 
-        Proof::new(metadata, pre_prepares, prepares, commits)
+        Proof::init_from_messages(metadata, messages_f)
     }
 
     fn init_proof_from_scm(metadata: DecisionMetadata<D, PBFTConsensus<D>>,
-                           messages: Vec<ShareableConsensusMessage<D, PBFTConsensus<D>>>) -> PProof<D, PBFTConsensus<D>, PBFTConsensus<D>> {
-        let mut pre_prepares = Vec::with_capacity(messages.len() / 2);
-        let mut prepares = Vec::with_capacity(messages.len() / 2);
-        let mut commits = Vec::with_capacity(messages.len() / 2);
-
-        for message in messages {
-            match message.message().consensus().kind() {
-                ConsensusMessageKind::PrePrepare(_) => {
-                    pre_prepares.push(message);
-                }
-                ConsensusMessageKind::Prepare(_) => {
-                    prepares.push(message);
-                }
-                ConsensusMessageKind::Commit(_) => {
-                    commits.push(message);
-                }
-            }
-        }
-
-        Proof::new(metadata, pre_prepares, prepares, commits)
+                           messages: Vec<ShareableConsensusMessage<D, PBFTConsensus<D>>>) -> Result<PProof<D, PBFTConsensus<D>, PBFTConsensus<D>>> {
+        Proof::init_from_messages(metadata, messages)
     }
 
     fn decompose_proof(proof: &Proof<D::Request>) -> (&ProofMetadata, Vec<&StoredMessage<PBFTMessage<D::Request>>>) {

From b42fc18dd5075c7bc689a0cda29d43ac79d7d8db Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Fri, 20 Oct 2023 20:30:10 +0100
Subject: [PATCH 67/80] add tracing to the channels so we know where we are
 having backup much faster

---
 febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 73ce87ca..20420ae5 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -16,17 +16,15 @@ use atlas_communication::message::Header;
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_core::persistent_log::OrderingProtocolLog;
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, TimeoutPhase, Timeouts};
-use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::{metric_duration, metric_increment};
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::consensus::Consensus;
 use crate::bft::log::decisions::CollectData;
 use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
-use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{SYNC_BATCH_RECEIVED_ID, SYNC_STOPPED_COUNT_ID, SYNC_STOPPED_REQUESTS_ID, SYNC_WATCH_REQUESTS_ID};
 use crate::bft::PBFT;
 use crate::bft::sync::view::ViewInfo;

From 4f9df07aa5017108644e01fd548e6408b4f4845a Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Mon, 23 Oct 2023 23:01:30 +0100
Subject: [PATCH 68/80] Worked on adding view transfer module necessary due to
 the now abstracted decision log (and permissioned ordering protocol) Also
 made Permissioned ordering protocol trait optional on SMR replica (same way
 as with reconfigurable, through specialization)

---
 febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
index 73ce87ca..20420ae5 100644
--- a/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/replica_sync/mod.rs
@@ -16,17 +16,15 @@ use atlas_communication::message::Header;
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_core::messages::{ClientRqInfo, ForwardedRequestsMessage, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
-use atlas_core::persistent_log::OrderingProtocolLog;
 use atlas_core::request_pre_processing::{PreProcessorMessage, RequestPreProcessor};
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, TimeoutPhase, Timeouts};
-use atlas_smr_application::serialize::ApplicationData;
 use atlas_metrics::metrics::{metric_duration, metric_increment};
+use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::consensus::Consensus;
 use crate::bft::log::decisions::CollectData;
 use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage, ViewChangeMessage, ViewChangeMessageKind};
-use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{SYNC_BATCH_RECEIVED_ID, SYNC_STOPPED_COUNT_ID, SYNC_STOPPED_REQUESTS_ID, SYNC_WATCH_REQUESTS_ID};
 use crate::bft::PBFT;
 use crate::bft::sync::view::ViewInfo;

From e27e728b17103f5616db830902c90edc638bf578 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Tue, 24 Oct 2023 18:52:48 +0100
Subject: [PATCH 69/80] Changed how the state machine of the SMR replica
 handles the novel protocol layout

---
 febft-state-transfer/src/lib.rs | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index c59e987a..6e5d9a48 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -23,7 +23,7 @@ use atlas_core::messages::StateTransfer;
 use atlas_core::ordering_protocol::{ExecutionResult, OrderingProtocol};
 use atlas_core::ordering_protocol::networking::serialize::NetworkView;
 use atlas_core::persistent_log::{MonolithicStateLog, OperationMode, PersistableStateTransferProtocol};
-use atlas_core::state_transfer::{Checkpoint, CstM, StateTransferProtocol, STResult, STTimeoutResult};
+use atlas_core::state_transfer::{Checkpoint, CstM, StateTransferProtocol, STPollResult, STResult, STTimeoutResult};
 use atlas_core::state_transfer::monolithic_state::MonolithicStateTransfer;
 use atlas_core::state_transfer::networking::StateTransferSendNode;
 use atlas_core::timeouts::{RqTimeout, TimeoutKind, Timeouts};
@@ -242,13 +242,15 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         Ok(())
     }
 
-    fn handle_off_ctx_message<V>(&mut self, view: V, message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
+    fn poll(&mut self) -> Result<STPollResult<CstM<Self::Serialization>>> {
+        Ok(STPollResult::ReceiveMsg)
+    }
+
+    fn handle_off_ctx_message<V>(&mut self, view: V, message: StoredMessage<CstM<Self::Serialization>>)
                                  -> Result<()>
         where V: NetworkView {
         let (header, message) = message.into_inner();
 
-        let message = message.into_inner();
-
         debug!("{:?} // Off context Message {:?} from {:?} with seq {:?}", self.node.id(), message, header.from(), message.sequence_number());
 
         match &message.kind() {
@@ -282,12 +284,10 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
     }
 
     fn process_message<V>(&mut self, view: V,
-                          message: StoredMessage<StateTransfer<CstM<Self::Serialization>>>)
+                          message: StoredMessage<CstM<Self::Serialization>>)
                           -> Result<STResult> where V: NetworkView {
         let (header, message) = message.into_inner();
 
-        let message = message.into_inner();
-
         debug!("{:?} // Message {:?} from {:?} while in phase {:?}", self.node.id(), message, header.from(), self.phase);
 
         match &message.kind() {
@@ -638,8 +638,6 @@ impl<S, NT, PL> CollabStateTransfer<S, NT, PL>
                                 received_state_cid.count += 1;
                             }
                         } else {
-
-
                             debug!("{:?} // Received blank state cid from node {:?}", self.node.id(), header.from());
                         }
                     }

From 17527e9f404864271204bc43aa93752a87078df0 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 25 Oct 2023 16:55:45 +0100
Subject: [PATCH 70/80] Fixed loads of compilation errors. Improved how
 specialization is handled.

---
 febft-pbft-consensus/src/bft/mod.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index fd7911ae..bf94faf3 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -128,6 +128,16 @@ impl<D, NT, > OrderProtocolTolerance for PBFTOrderProtocol<D, NT>
     fn get_n_for_f(f: usize) -> usize {
         3 * f + 1
     }
+
+    fn get_quorum_for_n(n: usize) -> usize {
+        //n = 2f+1
+
+        2 * Self::get_f_for_n(n) + 1
+    }
+
+    fn get_f_for_n(n: usize) -> usize {
+        (n - 1) / 3
+    }
 }
 
 impl<D, NT> OrderingProtocol<D, NT> for PBFTOrderProtocol<D, NT>

From 028080407b79045acbfc45046c6922d2efd1b7fc Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Fri, 27 Oct 2023 22:37:10 +0100
Subject: [PATCH 71/80] One compile error away from abstracting the state
 transfer from the protocol thread.

---
 febft-state-transfer/src/lib.rs | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 6e5d9a48..e1130835 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -403,16 +403,11 @@ impl<S, NT, PL> MonolithicStateTransfer<S, NT, PL> for CollabStateTransfer<S, NT
         Ok(Self::new(node, timeout_duration, timeouts, log, executor_handle))
     }
 
-    fn handle_state_received_from_app<V>(&mut self, view: V, state: Arc<ReadOnly<Checkpoint<S>>>) -> Result<()>
-        where V: NetworkView {
+    fn handle_state_received_from_app(&mut self, state: Arc<ReadOnly<Checkpoint<S>>>) -> Result<()> {
         self.finalize_checkpoint(state)?;
 
         if self.needs_checkpoint() {
-            // This will make the state transfer protocol aware of the latest state
-            if let CstStatus::Nil = self.process_message(view, CstProgress::Nil) {} else {
-                return Err("Process message while needing checkpoint returned something else than nil")
-                    .wrapped(ErrorKind::Cst);
-            }
+            self.process_pending_state_requests();
         }
 
         Ok(())

From 3ddacb69f759b24b903782254e10e1be18177be9 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Tue, 31 Oct 2023 16:55:19 +0000
Subject: [PATCH 72/80] Fixed all bugs, now successfully processing requests.

---
 .../src/bft/consensus/accessory/replica/mod.rs         | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index cba7cfb6..1644be87 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -87,8 +87,10 @@ impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
             }
         });
 
-        debug!("{:?} // Broadcasting prepare messages to quorum {:?}",
-               my_id, seq);
+        let targets = view.quorum_members().clone();
+
+        debug!("{:?} // Broadcasting prepare messages to quorum {:?}, {:?}",
+               my_id, seq, targets);
 
         // Vote for the received batch.
         // Leaders in this protocol must vote as they need to ack all
@@ -102,9 +104,7 @@ impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
             ConsensusMessageKind::Prepare(current_digest),
         ));
 
-        let targets = view.quorum_members().clone();
-
-        node.broadcast_signed(message, targets.into_iter());
+        node.broadcast_signed(message, targets.into_iter()).expect("Failed to send");
     }
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,

From 5eca0154f39e555115b0f4ed9156a08c288522b7 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Sun, 12 Nov 2023 19:20:29 +0000
Subject: [PATCH 73/80] Working on further separating the decision log from the
 and therefore achieve full separation

---
 febft-state-transfer/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index e1130835..43c1a766 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -348,7 +348,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         Ok(STResult::StateTransferRunning)
     }
 
-    fn handle_app_state_requested<V>(&mut self, view: V, seq: SeqNo) -> Result<ExecutionResult>
+    fn handle_app_state_requested<V>(&mut self, seq: SeqNo) -> Result<ExecutionResult>
         where V: NetworkView {
         let earlier = std::mem::replace(&mut self.current_checkpoint_state, CheckpointState::None);
 

From 1a14e098fd0a8e915d59fc50ea98fa8088735ab0 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Tue, 14 Nov 2023 19:15:52 +0000
Subject: [PATCH 74/80] Finished abstracting the Decision log and log transfer
 protocols and running them on their own thread. Now onto testing.

---
 .../src/bft/consensus/accessory/replica/mod.rs                 | 2 +-
 febft-state-transfer/src/lib.rs                                | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
index 1644be87..d9c78a20 100644
--- a/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/accessory/replica/mod.rs
@@ -104,7 +104,7 @@ impl<D> AccessoryConsensus<D> for ReplicaAccessory<D>
             ConsensusMessageKind::Prepare(current_digest),
         ));
 
-        node.broadcast_signed(message, targets.into_iter()).expect("Failed to send");
+        node.broadcast_signed(message, targets.into_iter());
     }
 
     fn handle_preparing_no_quorum<NT>(&mut self, deciding_log: &WorkingDecisionLog<D::Request>,
diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 43c1a766..7074538a 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -348,8 +348,7 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
         Ok(STResult::StateTransferRunning)
     }
 
-    fn handle_app_state_requested<V>(&mut self, seq: SeqNo) -> Result<ExecutionResult>
-        where V: NetworkView {
+    fn handle_app_state_requested(&mut self, seq: SeqNo) -> Result<ExecutionResult> {
         let earlier = std::mem::replace(&mut self.current_checkpoint_state, CheckpointState::None);
 
         self.current_checkpoint_state = match earlier {

From 98ad41bbdd99885920296be9d0cd0479c92116f2 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 15 Nov 2023 01:08:18 +0000
Subject: [PATCH 75/80] Fixed some more issues with the parallel decision log,
 still having some issues with actually running it.

---
 febft-state-transfer/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/febft-state-transfer/src/lib.rs b/febft-state-transfer/src/lib.rs
index 7074538a..851eb20d 100644
--- a/febft-state-transfer/src/lib.rs
+++ b/febft-state-transfer/src/lib.rs
@@ -327,7 +327,8 @@ impl<S, NT, PL> StateTransferProtocol<S, NT, PL> for CollabStateTransfer<S, NT,
 
                     self.request_latest_state(view);
                 } else {
-                    debug!("{:?} // Not installing sequence number nor requesting state ???? {:?} {:?}", self.node.id(), self.current_checkpoint_state.sequence_number(), seq);
+                    debug!("{:?} // Not installing sequence number nor requesting state {:?} {:?}", self.node.id(), self.current_checkpoint_state.sequence_number(), seq);
+
                     return Ok(STResult::StateTransferNotNeeded(seq));
                 }
             }

From 7ff93493540673b2e9afb528b3dee9a105581df3 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 15 Nov 2023 03:05:29 +0000
Subject: [PATCH 76/80] Fixed issue leading to the decision log to not
 processing backlogged messages from running log transfers. Fixed some other
 issues with febft's transition to this style

---
 febft-pbft-consensus/src/bft/consensus/mod.rs |   9 +-
 febft-pbft-consensus/src/bft/mod.rs           | 166 +++++++++---------
 2 files changed, 90 insertions(+), 85 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 1815c662..2a27fae2 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -11,9 +11,9 @@ use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::maybe_vec::MaybeVec;
 use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_advance_message_queue_return, tbo_queue_message, tbo_queue_message_arc};
+use atlas_common::ordering::{Orderable, SeqNo, tbo_advance_message_queue, tbo_advance_message_queue_return, tbo_queue_message_arc};
 use atlas_communication::message::{Header, StoredMessage};
-use atlas_core::messages::{ClientRqInfo, RequestMessage, StoredRequestMessage};
+use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
 use atlas_core::ordering_protocol::Decision;
 use atlas_core::ordering_protocol::networking::OrderProtocolSendNode;
 use atlas_core::smr::smr_decision_log::ShareableMessage;
@@ -24,13 +24,12 @@ use atlas_smr_application::serialize::ApplicationData;
 
 use crate::bft::{OPDecision, PBFT, SysMsg};
 use crate::bft::consensus::decision::{ConsensusDecision, DecisionPollStatus, DecisionStatus, MessageQueue};
-use crate::bft::log::decided::DecisionLog;
 use crate::bft::log::deciding::CompletedBatch;
 use crate::bft::log::decisions::{IncompleteProof, Proof, ProofMetadata};
 use crate::bft::log::Log;
 use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
 use crate::bft::metric::OPERATIONS_PROCESSED_ID;
-use crate::bft::sync::{Synchronizer};
+use crate::bft::sync::Synchronizer;
 use crate::bft::sync::view::ViewInfo;
 
 pub mod decision;
@@ -433,6 +432,8 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
                 // That were in the queue, so we must be signalled again
                 self.signalled.push_signalled(decision_seq);
 
+                debug!("Received transitioned state with metadata {:?}", metadata);
+
                 if let Some(metadata) = metadata {
                     ConsensusStatus::Deciding(MaybeVec::from_one(Decision::decision_info_from_metadata_and_messages(decision_seq, metadata, MaybeVec::from_one(message))))
                 } else {
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index bf94faf3..53706927 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -449,11 +449,7 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
                 OPPollResult::Exec(message)
             }
             ConsensusPollStatus::Decided(decisions) => {
-                let finalized_decisions = self.finalize_all_possible()?;
-
-                let decisions = self.merge_decisions(decisions, finalized_decisions)?;
-
-                OPPollResult::ProgressedDecision(DecisionsAhead::Ignore, decisions)
+                OPPollResult::ProgressedDecision(DecisionsAhead::Ignore, self.handle_decided(decisions)?)
             }
         })
     }
@@ -500,80 +496,14 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
         Ok(OPExecResult::MessageProcessedNoUpdate)
     }
 
-    fn merge_decisions(&mut self, status: MaybeVec<OPDecision<D::Request>>, finalized_decisions: Vec<ProtocolConsensusDecision<D::Request>>) -> Result<MaybeVec<OPDecision<D::Request>>> {
-        let mut map = BTreeMap::new();
-
-        Self::merge_decision_vec(&mut map, status)?;
-
-        for decision in finalized_decisions {
-            if let Some(member) = map.get_mut(&decision.sequence_number()) {
-                member.append_decision_info(DecisionInfo::DecisionDone(decision));
-            } else {
-                map.insert(decision.sequence_number(), Decision::completed_decision(decision.sequence_number(), decision));
-            }
-        }
-
-        let mut decisions = MaybeVec::builder();
-
-        // By turning this btree map into a vec, we maintain ordering on the delivery (Shouldn't
-        // really be necessary but always nice to have)
-        map.into_iter().for_each(|(seq, decision)| {
-            decisions.push(decision);
-        });
-
-        Ok(decisions.build())
-    }
-
-    /// Handles the result of a synchronizer result
-    fn handle_sync_result(&mut self, status: ConsensusStatus<D::Request>, to_exec: Option<OPDecision<D::Request>>) -> Result<MaybeVec<OPDecision<D::Request>>> {
-        let mut map = BTreeMap::new();
-
-        match status {
-            ConsensusStatus::Deciding(decision) => {
-                Self::merge_decision_vec(&mut map, decision)?;
-            }
-            ConsensusStatus::Decided(decision) => {
-                Self::merge_decision_vec(&mut map, decision)?;
-
-                let finalized = self.finalize_all_possible()?;
-
-                for decision in finalized {
-                    if let Some(member) = map.get_mut(&decision.sequence_number()) {
-                        member.append_decision_info(DecisionInfo::DecisionDone(decision));
-                    } else {
-                        map.insert(decision.sequence_number(), Decision::completed_decision(decision.sequence_number(), decision));
-                    }
-                }
-            }
-            _ => {}
-        }
-
-        if let Some(decision) = to_exec {
-            map.insert(decision.sequence_number(), decision);
-        }
-
-        let mut decisions = MaybeVec::builder();
-
-        // By turning this btree map into a vec, we maintain ordering on the delivery (Shouldn't
-        // really be necessary but always nice to have)
-        map.into_iter().for_each(|(seq, decision)| {
-            decisions.push(decision);
-        });
+    fn handle_decided(&mut self, decisions: MaybeVec<Decision<ProofMetadata, PBFTMessage<D::Request>, D::Request>>) -> Result<MaybeVec<OPDecision<D::Request>>> {
+        let finalized_decisions = self.finalize_all_possible()?;
 
-        Ok(decisions.build())
-    }
+        debug!("Received decided decisions {:?}, merging with finalized decisions {:?}", decisions, finalized_decisions);
 
-    /// Merge a decision vector with the already existing btreemap
-    fn merge_decision_vec(map: &mut BTreeMap<SeqNo, OPDecision<<D as ApplicationData>::Request>>, decision: MaybeVec<OPDecision<<D as ApplicationData>::Request>>) -> Result<()> {
-        for dec in decision.into_iter() {
-            if let Some(member) = map.get_mut(&dec.sequence_number()) {
-                member.merge_decisions(dec)?;
-            } else {
-                map.insert(dec.sequence_number(), dec);
-            }
-        }
+        let decisions = self.merge_decisions(decisions, finalized_decisions)?;
 
-        Ok(())
+        Ok(decisions)
     }
 
     fn update_normal_phase(&mut self, message: ShareableMessage<PBFTMessage<D::Request>>) -> Result<OPExecResult<ProofMetadata, PBFTMessage<D::Request>, D::Request>> {
@@ -641,11 +571,7 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
                 OPExecResult::ProgressedDecision(DecisionsAhead::Ignore, result)
             }
             ConsensusStatus::Decided(result) => {
-                let finalized_decisions = self.finalize_all_possible()?;
-
-                let decision = self.merge_decisions(result, finalized_decisions)?;
-
-                OPExecResult::ProgressedDecision(DecisionsAhead::Ignore, decision)
+                OPExecResult::ProgressedDecision(DecisionsAhead::Ignore, self.handle_decided(result)?)
             }
         });
     }
@@ -721,6 +647,84 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
             }
         };
     }
+
+    fn merge_decisions(&mut self, status: MaybeVec<OPDecision<D::Request>>, finalized_decisions: Vec<ProtocolConsensusDecision<D::Request>>) -> Result<MaybeVec<OPDecision<D::Request>>> {
+        let mut map = BTreeMap::new();
+
+        debug!("Merging the decisions {:?} with finalized decisions {:?}", status, finalized_decisions);
+
+        Self::merge_decision_vec(&mut map, status)?;
+
+        for decision in finalized_decisions {
+            if let Some(member) = map.get_mut(&decision.sequence_number()) {
+                member.append_decision_info(DecisionInfo::DecisionDone(decision));
+            } else {
+                map.insert(decision.sequence_number(), Decision::completed_decision(decision.sequence_number(), decision));
+            }
+        }
+
+        let mut decisions = MaybeVec::builder();
+
+        // By turning this btree map into a vec, we maintain ordering on the delivery (Shouldn't
+        // really be necessary but always nice to have)
+        map.into_iter().for_each(|(seq, decision)| {
+            decisions.push(decision);
+        });
+
+        Ok(decisions.build())
+    }
+
+    /// Handles the result of a synchronizer result
+    fn handle_sync_result(&mut self, status: ConsensusStatus<D::Request>, to_exec: Option<OPDecision<D::Request>>) -> Result<MaybeVec<OPDecision<D::Request>>> {
+        let mut map = BTreeMap::new();
+
+        match status {
+            ConsensusStatus::Deciding(decision) => {
+                Self::merge_decision_vec(&mut map, decision)?;
+            }
+            ConsensusStatus::Decided(decision) => {
+                Self::merge_decision_vec(&mut map, decision)?;
+
+                let finalized = self.finalize_all_possible()?;
+
+                for decision in finalized {
+                    if let Some(member) = map.get_mut(&decision.sequence_number()) {
+                        member.append_decision_info(DecisionInfo::DecisionDone(decision));
+                    } else {
+                        map.insert(decision.sequence_number(), Decision::completed_decision(decision.sequence_number(), decision));
+                    }
+                }
+            }
+            _ => {}
+        }
+
+        if let Some(decision) = to_exec {
+            map.insert(decision.sequence_number(), decision);
+        }
+
+        let mut decisions = MaybeVec::builder();
+
+        // By turning this btree map into a vec, we maintain ordering on the delivery (Shouldn't
+        // really be necessary but always nice to have)
+        map.into_iter().for_each(|(seq, decision)| {
+            decisions.push(decision);
+        });
+
+        Ok(decisions.build())
+    }
+
+    /// Merge a decision vector with the already existing btreemap
+    fn merge_decision_vec(map: &mut BTreeMap<SeqNo, OPDecision<<D as ApplicationData>::Request>>, decision: MaybeVec<OPDecision<<D as ApplicationData>::Request>>) -> Result<()> {
+        for dec in decision.into_iter() {
+            if let Some(member) = map.get_mut(&dec.sequence_number()) {
+                member.merge_decisions(dec)?;
+            } else {
+                map.insert(dec.sequence_number(), dec);
+            }
+        }
+
+        Ok(())
+    }
 }
 
 impl<D, NT> PBFTOrderProtocol<D, NT>

From 3523519fae2f8f3aca8e62502fe8e1cd5b6052cf Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 15 Nov 2023 03:12:58 +0000
Subject: [PATCH 77/80] Removed unnecessary debugging

---
 febft-pbft-consensus/src/bft/consensus/mod.rs | 2 --
 febft-pbft-consensus/src/bft/mod.rs           | 4 ----
 2 files changed, 6 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 2a27fae2..92c89d94 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -432,8 +432,6 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
                 // That were in the queue, so we must be signalled again
                 self.signalled.push_signalled(decision_seq);
 
-                debug!("Received transitioned state with metadata {:?}", metadata);
-
                 if let Some(metadata) = metadata {
                     ConsensusStatus::Deciding(MaybeVec::from_one(Decision::decision_info_from_metadata_and_messages(decision_seq, metadata, MaybeVec::from_one(message))))
                 } else {
diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 53706927..0d363c0e 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -499,8 +499,6 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
     fn handle_decided(&mut self, decisions: MaybeVec<Decision<ProofMetadata, PBFTMessage<D::Request>, D::Request>>) -> Result<MaybeVec<OPDecision<D::Request>>> {
         let finalized_decisions = self.finalize_all_possible()?;
 
-        debug!("Received decided decisions {:?}, merging with finalized decisions {:?}", decisions, finalized_decisions);
-
         let decisions = self.merge_decisions(decisions, finalized_decisions)?;
 
         Ok(decisions)
@@ -651,8 +649,6 @@ impl<D, NT> PBFTOrderProtocol<D, NT>
     fn merge_decisions(&mut self, status: MaybeVec<OPDecision<D::Request>>, finalized_decisions: Vec<ProtocolConsensusDecision<D::Request>>) -> Result<MaybeVec<OPDecision<D::Request>>> {
         let mut map = BTreeMap::new();
 
-        debug!("Merging the decisions {:?} with finalized decisions {:?}", status, finalized_decisions);
-
         Self::merge_decision_vec(&mut map, status)?;
 
         for decision in finalized_decisions {

From 73431505a977953739e35138dc3cca644c2e6b28 Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Wed, 15 Nov 2023 17:21:45 +0000
Subject: [PATCH 78/80] Fixed rejoining the quorum after crashing.

---
 febft-pbft-consensus/src/bft/consensus/mod.rs | 6 +++++-
 febft-pbft-consensus/src/bft/sync/mod.rs      | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/consensus/mod.rs b/febft-pbft-consensus/src/bft/consensus/mod.rs
index 92c89d94..88b7932e 100644
--- a/febft-pbft-consensus/src/bft/consensus/mod.rs
+++ b/febft-pbft-consensus/src/bft/consensus/mod.rs
@@ -608,7 +608,7 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
                 self.seq_no = novel_seq_no;
             }
             Either::Right(limit) => {
-                debug!("{:?} // Installed sequence number is right of the current one and is smaller than the decisions we have stored. Removing decided decisions.", self.node_id);
+                debug!("{:?} // Installed sequence number is right of the current one and is smaller than the decisions we have stored. Removing decided decisions until sequence {:?}", self.node_id, novel_seq_no);
 
                 for _ in 0..limit {
                     // Pop the decisions that have already been made and dispose of them
@@ -621,6 +621,8 @@ impl<D> Consensus<D> where D: ApplicationData + 'static {
                 // The following new consensus decisions will have the sequence number of the last decision
                 let mut sequence_no: SeqNo = self.decisions.back().unwrap().sequence_number().next();
 
+                debug!("Repopulating decision vec until we reach watermark. Current seq {:?}, current decision len {}, watermark {}", sequence_no, self.decisions.len(), self.watermark);
+
                 while self.decisions.len() < self.watermark as usize {
                     // We advanced [`limit`] sequence numbers on the decisions,
                     // so by advancing the tbo queue the missing decisions, we will
@@ -905,6 +907,8 @@ impl ProposerConsensusGuard {
         while let Some(seq) = guard.0.peek() {
             if seq.0 < installed_seq {
                 guard.0.pop();
+            } else {
+                break;
             }
         }
     }
diff --git a/febft-pbft-consensus/src/bft/sync/mod.rs b/febft-pbft-consensus/src/bft/sync/mod.rs
index ae9109e4..60bc9b94 100755
--- a/febft-pbft-consensus/src/bft/sync/mod.rs
+++ b/febft-pbft-consensus/src/bft/sync/mod.rs
@@ -524,12 +524,12 @@ impl<D> AbstractSynchronizer<D> for Synchronizer<D>
             }
 
             self.install_next_view(view);
+
             return true;
         } else {
             // This is the first view, so we can just install it
             if !self.tbo.lock().unwrap().install_view(view) {
                 // If we don't install a new view, then we don't want to forget our current state now do we?
-
                 debug!("Replacing our phase with Init");
                 self.phase.replace(ProtoPhase::Init);
             }

From aa18cbe4b9b250af25054f4e031ffa0429b1885d Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 16 Nov 2023 16:09:33 +0000
Subject: [PATCH 79/80] Fix issue with view changing.

---
 .../src/bft/log/deciding/mod.rs               | 56 ++++++++++++++++---
 1 file changed, 49 insertions(+), 7 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/log/deciding/mod.rs b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
index 2cf519fa..1f302bf3 100644
--- a/febft-pbft-consensus/src/bft/log/deciding/mod.rs
+++ b/febft-pbft-consensus/src/bft/log/deciding/mod.rs
@@ -2,18 +2,19 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::iter;
 use std::sync::{Arc, Mutex};
 use std::time::Instant;
+
 use atlas_common::crypto::hash::{Context, Digest};
+use atlas_common::error::*;
 use atlas_common::node_id::NodeId;
 use atlas_common::ordering::{Orderable, SeqNo};
-use atlas_common::error::*;
-use atlas_communication::message::Header;
 use atlas_core::messages::{ClientRqInfo, StoredRequestMessage};
 use atlas_core::ordering_protocol::networking::serialize::NetworkView;
 use atlas_core::smr::smr_decision_log::ShareableMessage;
 use atlas_metrics::benchmarks::BatchMeta;
 use atlas_metrics::metrics::metric_duration;
-use crate::bft::log::decisions::{IncompleteProof, ProofMetadata};
-use crate::bft::message::{ConsensusMessage, ConsensusMessageKind, PBFTMessage};
+
+use crate::bft::log::decisions::{IncompleteProof, PrepareSet, ProofMetadata, ViewDecisionPair};
+use crate::bft::message::{ConsensusMessageKind, PBFTMessage};
 use crate::bft::metric::PRE_PREPARE_LOG_ANALYSIS_ID;
 use crate::bft::sync::view::ViewInfo;
 
@@ -270,10 +271,52 @@ impl<O> WorkingDecisionLog<O> where O: Clone {
         Some((ctx.finish(), batch_ordered_digests))
     }
 
-
     /// Get the current decision
     pub fn deciding(&self, f: usize) -> IncompleteProof {
-        todo!()
+        let write_set = PrepareSet({
+            let mut set = Vec::new();
+
+            for shareable_msg in self.message_log.prepares.iter().rev() {
+                let digest = match shareable_msg.message().consensus().kind() {
+                    ConsensusMessageKind::Prepare(d) => d.clone(),
+                    _ => unreachable!(),
+                };
+
+                set.push(ViewDecisionPair(
+                    shareable_msg.message().consensus().view(),
+                    digest,
+                ));
+            }
+
+            set
+        });
+
+        let quorum_prepares = 'outer: loop {
+            let quorum = f << 1;
+            let mut last_view = None;
+            let mut count = 0;
+
+            for stored in self.message_log.prepares.iter().rev() {
+                match last_view {
+                    None => (),
+                    Some(v) if stored.message().consensus().view() == v => (),
+                    _ => count = 0,
+                }
+                last_view = Some(stored.message().consensus().view());
+                count += 1;
+                if count == quorum {
+                    let digest = match stored.message().consensus().kind() {
+                        ConsensusMessageKind::Prepare(d) => d.clone(),
+                        _ => unreachable!(),
+                    };
+                    break 'outer Some(ViewDecisionPair(stored.message().consensus().view(), digest));
+                }
+            }
+
+            break 'outer None;
+        };
+
+        IncompleteProof::new(self.seq_no, write_set, quorum_prepares)
     }
 
     /// Indicate that the batch is finished processing and
@@ -328,7 +371,6 @@ impl<O> CompletedBatch<O> {
     pub fn request_count(&self) -> usize {
         self.client_requests.len()
     }
-    
 }
 
 impl<O> Orderable for WorkingDecisionLog<O> {

From 61e479ba7ec40335a6e3ffed74c35aa60a68d01c Mon Sep 17 00:00:00 2001
From: Nuno Neto <nuno.neto.g@gmail.com>
Date: Thu, 16 Nov 2023 16:36:26 +0000
Subject: [PATCH 80/80] Fixed view transfer not correctly installing view into
 consensus.

---
 febft-pbft-consensus/src/bft/mod.rs | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/febft-pbft-consensus/src/bft/mod.rs b/febft-pbft-consensus/src/bft/mod.rs
index 0d363c0e..85fec171 100644
--- a/febft-pbft-consensus/src/bft/mod.rs
+++ b/febft-pbft-consensus/src/bft/mod.rs
@@ -10,12 +10,13 @@ use std::sync::Arc;
 use std::sync::atomic::AtomicBool;
 use std::time::Instant;
 use ::log::{debug, error, info, trace, warn};
+use either::Either;
 
 use atlas_common::error::*;
 use atlas_common::globals::ReadOnly;
 use atlas_common::maybe_vec::MaybeVec;
 use atlas_common::node_id::NodeId;
-use atlas_common::ordering::{Orderable, SeqNo};
+use atlas_common::ordering::{InvalidSeqNo, Orderable, SeqNo};
 use atlas_communication::message::{Header, StoredMessage};
 use atlas_communication::protocol_node::ProtocolNetworkNode;
 use atlas_communication::serialize::Serializable;
@@ -45,6 +46,7 @@ use crate::bft::message::serialize::PBFTConsensus;
 use crate::bft::metric::{CONSENSUS_INSTALL_STATE_TIME_ID, MSG_LOG_INSTALL_TIME_ID};
 use crate::bft::proposer::Proposer;
 use crate::bft::sync::{AbstractSynchronizer, Synchronizer, SynchronizerPollStatus, SynchronizerStatus, SyncReconfigurationResult};
+use crate::bft::sync::view::ViewInfo;
 
 pub mod consensus;
 pub mod proposer;
@@ -260,13 +262,25 @@ impl<D, NT> PermissionedOrderingProtocol for PBFTOrderProtocol<D, NT>
           NT: OrderProtocolSendNode<D, PBFT<D>> + 'static {
     type PermissionedSerialization = PBFTConsensus<D>;
 
-    fn view(&self) -> View<Self::PermissionedSerialization> {
+    fn view(&self) -> ViewInfo {
         self.synchronizer.view()
     }
 
-    fn install_view(&mut self, view: View<Self::PermissionedSerialization>) {
-        if self.synchronizer.received_view_from_state_transfer(view.clone()) {
-            self.consensus.install_view(&view)
+    fn install_view(&mut self, view: ViewInfo) {
+        let current_view = self.view();
+
+        match view.sequence_number().index(current_view.sequence_number()) {
+            Either::Left(_) | Either::Right(0) => {
+                warn!("Attempted to install view that is the same or older than the current view that is in place? New: {:?} vs {:?}", view, current_view);
+            }
+            Either::Right(_) => {
+                self.consensus.install_view(&view);
+                if self.synchronizer.received_view_from_state_transfer(view) {
+                    info!("Installed the view and synchronizer now requires execution in order to make sure everything is correctly setup.");
+
+                    self.switch_phase(ConsensusPhase::SyncPhase);
+                }
+            }
         }
     }
 }