Playback via ledger entries

CMakeLists.txt

@@ -601,6 +601,12 @@ if(BUILD_TESTS)
     ADDITIONAL_ARGS --election-timeout 2000
   )
 
+  add_e2e_test(
+    NAME late_joiners
+    PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/late_joiners.py
+    ADDITIONAL_ARGS --skip-suspension
+  )
+
   if(NOT PBFT)
     # Logging scenario perf test
     add_perf_test(

cmake/pbft.cmake

@@ -5,6 +5,11 @@
 add_definitions(-DSIGN_BATCH)
 set(SIGN_BATCH ON)
 
+if(SAN)
+  add_definitions(-DUSE_STD_MALLOC)
+  set(USE_STD_MALLOC ON)
+endif()
+
 set(PBFT_SRC
     ${CMAKE_SOURCE_DIR}/src/consensus/pbft/libbyz/globalstate.cpp
     ${CMAKE_SOURCE_DIR}/src/consensus/pbft/libbyz/Client.cpp
@@ -47,6 +52,7 @@ set(PBFT_SRC
     ${CMAKE_SOURCE_DIR}/src/consensus/pbft/libbyz/request_id_gen.cpp
     ${CMAKE_SOURCE_DIR}/src/consensus/pbft/libbyz/New_principal.cpp
     ${CMAKE_SOURCE_DIR}/src/consensus/pbft/libbyz/Network_open.cpp
+    ${CMAKE_SOURCE_DIR}/src/consensus/pbft/libbyz/Append_entries.cpp
 )
 
 if("sgx" IN_LIST TARGET)
@@ -114,6 +120,7 @@ if("virtual" IN_LIST TARGET)
             ${CMAKE_SOURCE_DIR}/src/consensus/pbft/libbyz/test ${EVERCRYPT_INC}
   )
 
+  add_dependencies(libcommontest.mock flatbuffers)
   target_compile_options(libcommontest.mock PRIVATE -stdlib=libc++)
 
   function(use_libbyz name)

src/consensus/consensustypes.h

@@ -7,4 +7,24 @@ namespace ccf
 {
   using ObjectId = uint64_t;
   using NodeId = ObjectId;
+  using Index = int64_t;
+  using Node2NodeMsg = uint64_t;
+}
+
+namespace consensus
+{
+#pragma pack(push, 1)
+  template <typename T = ccf::Node2NodeMsg>
+  struct ConsensusHeader
+  {
+    T msg;
+    ccf::NodeId from_node;
+  };
+
+  struct AppendEntriesIndex
+  {
+    ccf::Index idx;
+    ccf::Index prev_idx;
+  };
+#pragma pack(pop)
 }

src/consensus/ledgerenclave.h

@@ -83,6 +83,17 @@ namespace consensus
       serialized::skip(data, size, entry_len);
     }
 
+    std::pair<std::vector<uint8_t>, bool> get_entry(
+      const uint8_t*& data, size_t& size)
+    {
+      auto entry_len = serialized::read<uint32_t>(data, size);
+      std::vector<uint8_t> entry(data, data + entry_len);
+
+      serialized::skip(data, size, entry_len);
+
+      return std::make_pair(std::move(entry), true);
+    }
+
     /**
      * Truncate the ledger at a given index.
      *

src/consensus/pbft/libbyz/Append_entries.cpp

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation.
+// Copyright (c) 1999 Miguel Castro, Barbara Liskov.
+// Copyright (c) 2000, 2001 Miguel Castro, Rodrigo Rodrigues, Barbara Liskov.
+// Licensed under the MIT license.
+#include "Append_entries.h"
+
+#include "Message_tags.h"
+#include "ds/logger.h"
+#include "pbft_assert.h"
+
+Append_entries::Append_entries() :
+  Message(Append_entries_tag, sizeof(Append_entries_rep))
+{}
+
+bool Append_entries::verify()
+{
+  return true;
+}
+
+bool Append_entries::convert(Message* m1, Append_entries*& m2)
+{
+  if (!m1->has_tag(Append_entries_tag, sizeof(Append_entries_rep)))
+  {
+    return false;
+  }
+
+  m1->trim();
+  m2 = (Append_entries*)m1;
+  return true;
+}
+
+Append_entries_rep& Append_entries::rep() const
+{
+  PBFT_ASSERT(ALIGNED(msg), "Improperly aligned pointer");
+  return *((Append_entries_rep*)msg);
+}

src/consensus/pbft/libbyz/Append_entries.h

@@ -0,0 +1,30 @@
+// Copyright (c) Microsoft Corporation.
+// Copyright (c) 1999 Miguel Castro, Barbara Liskov.
+// Copyright (c) 2000, 2001 Miguel Castro, Rodrigo Rodrigues, Barbara Liskov.
+// Licensed under the MIT license.
+#pragma once
+
+#include "Message.h"
+#include "nodeinfo.h"
+
+//
+// Append entries messages have the following format:
+//
+#pragma pack(push)
+#pragma pack(1)
+struct Append_entries_rep : public Message_rep
+{};
+#pragma pack(pop)
+
+class Append_entries : public Message
+{
+public:
+  Append_entries();
+
+  bool verify();
+
+  static bool convert(Message* m1, Append_entries*& m2);
+
+private:
+  Append_entries_rep& rep() const;
+};

src/consensus/pbft/libbyz/Certificate.h

@@ -347,6 +347,16 @@ void Certificate<T>::reset_f()
 template <class T>
 bool Certificate<T>::add(T* m)
 {
+  auto principal = pbft::GlobalState::get_node().get_principal(m->id());
+  if (!principal)
+  {
+    LOG_INFO_FMT(
+      "Principal with id {} has not been configured yet, rejecting the message",
+      m->id());
+    delete m;
+    return false;
+  }
+
   if (bmap.none() && f != pbft::GlobalState::get_node().f())
   {
     reset_f();

src/consensus/pbft/libbyz/LedgerWriter.cpp

@@ -5,7 +5,7 @@
 #include "Request.h"
 
 LedgerWriter::LedgerWriter(
-  pbft::Store& store_, pbft::PrePreparesMap& pbft_pre_prepares_map_) :
+  pbft::PbftStore& store_, pbft::PrePreparesMap& pbft_pre_prepares_map_) :
   store(store_),
   pbft_pre_prepares_map(pbft_pre_prepares_map_)
 {}
@@ -35,14 +35,21 @@ void LedgerWriter::write_prepare(
 }
 
 void LedgerWriter::write_pre_prepare(Pre_prepare* pp)
+{
+  ccf::Store::Tx tx;
+  write_pre_prepare(pp, tx);
+}
+
+void LedgerWriter::write_pre_prepare(Pre_prepare* pp, ccf::Store::Tx& tx)
 {
   store.commit_pre_prepare(
     {pp->seqno(),
      pp->num_big_reqs(),
      pp->get_digest_sig(),
      {(const uint8_t*)pp->contents(),
       (const uint8_t*)pp->contents() + pp->size()}},
-    pbft_pre_prepares_map);
+    pbft_pre_prepares_map,
+    tx);
 }
 
 void LedgerWriter::write_view_change(View_change* vc)

src/consensus/pbft/libbyz/LedgerWriter.h

@@ -14,14 +14,15 @@
 class LedgerWriter
 {
 private:
-  pbft::Store& store;
+  pbft::PbftStore& store;
   pbft::PrePreparesMap& pbft_pre_prepares_map;
 
 public:
   LedgerWriter(
-    pbft::Store& store_, pbft::PrePreparesMap& pbft_pre_prepares_map_);
+    pbft::PbftStore& store_, pbft::PrePreparesMap& pbft_pre_prepares_map_);
   virtual ~LedgerWriter() = default;
   void write_prepare(const Prepared_cert& prepared_cert, Seqno seqno);
   void write_pre_prepare(Pre_prepare* pp);
+  void write_pre_prepare(Pre_prepare* pp, ccf::Store::Tx& tx);
   void write_view_change(View_change* vc);
 };

src/consensus/pbft/libbyz/Message_tags.h

@@ -29,7 +29,8 @@ const short Fetch_tag = 15;
 const short Query_stable_tag = 16;
 const short Reply_stable_tag = 17;
 const short Network_open_tag = 18;
-const short Max_message_tag = 19;
+const short Append_entries_tag = 19;
+const short Max_message_tag = 20;
 
 // Message used for testing are in the 100+ range
 const short New_principal_tag = 100;

src/consensus/pbft/libbyz/Pre_prepare.cpp

@@ -289,6 +289,11 @@ bool Pre_prepare::pre_verify()
     {
       auto sender_principal =
         pbft::GlobalState::get_node().get_principal(sender);
+      if (!sender_principal)
+      {
+        LOG_INFO_FMT("Sender principal has not been configured yet {}", sender);
+        return false;
+      }
 
       if (
         !sender_principal->has_certificate_set() &&

src/consensus/pbft/libbyz/Prepared_cert.h

@@ -140,8 +140,18 @@ class Prepared_cert
 
 inline bool Prepared_cert::add(Prepare* m)
 {
-#ifdef SIGN_BATCH
   int id = m->id();
+  auto principal = pbft::GlobalState::get_node().get_principal(id);
+  if (!principal)
+  {
+    LOG_INFO_FMT(
+      "Principal with id {} has not been configured yet, rejecting prepare",
+      id);
+    delete m;
+    return false;
+  }
+
+#ifdef SIGN_BATCH
   PbftSignature& digest_sig = m->digest_sig();
   PrePrepareProof proof;
   std::copy(
@@ -153,7 +163,7 @@ inline bool Prepared_cert::add(Prepare* m)
 #ifdef SIGN_BATCH
   if (result)
   {
-    proof.cert = pbft::GlobalState::get_node().get_principal(id)->get_cert();
+    proof.cert = principal->get_cert();
     pre_prepare_proof.insert({id, proof});
   }
 #endif