Proposed 1.6.0-b6

Builds/CMake/RippleConfig.cmake

@@ -46,6 +46,7 @@ endif ()
 set (OPENSSL_MSVC_STATIC_RT ON)
 find_dependency (OpenSSL 1.0.2 REQUIRED)
 find_dependency (ZLIB)
+find_dependency (date)
 if (TARGET ZLIB::ZLIB)
   set_target_properties(OpenSSL::Crypto PROPERTIES
     INTERFACE_LINK_LIBRARIES ZLIB::ZLIB)

Builds/CMake/RippledCore.cmake

@@ -622,6 +622,7 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/impl/Role.cpp
   src/ripple/rpc/impl/ServerHandlerImp.cpp
   src/ripple/rpc/impl/ShardArchiveHandler.cpp
+  src/ripple/rpc/impl/ShardVerificationScheduler.cpp
   src/ripple/rpc/impl/Status.cpp
   src/ripple/rpc/impl/TransactionSign.cpp
 
@@ -836,7 +837,7 @@ target_sources (rippled PRIVATE
      test sources:
        subdir: net
   #]===============================]
-  src/test/net/SSLHTTPDownloader_test.cpp
+  src/test/net/DatabaseDownloader_test.cpp
   #[===============================[
      test sources:
        subdir: nodestore

Builds/CMake/deps/gRPC.cmake

@@ -63,20 +63,23 @@ else ()
     if (cares_FOUND)
       if (static)
         set (_search "${CMAKE_STATIC_LIBRARY_PREFIX}cares${CMAKE_STATIC_LIBRARY_SUFFIX}")
+        set (_prefix cares_STATIC)
+        set (_static STATIC)
       else ()
         set (_search "${CMAKE_SHARED_LIBRARY_PREFIX}cares${CMAKE_SHARED_LIBRARY_SUFFIX}")
+        set (_prefix cares)
+        set (_static)
       endif()
-      find_library(_cares
-        NAMES ${_search}
-        HINTS ${cares_LIBRARY_DIRS})
-      if (NOT _cares)
+      find_library(_location NAMES ${_search} HINTS ${cares_LIBRARY_DIRS})
+      if (NOT _location)
         message (FATAL_ERROR "using pkg-config for grpc, can't find c-ares")
       endif ()
-      add_library (c-ares::cares STATIC IMPORTED GLOBAL)
-      set_target_properties (c-ares::cares PROPERTIES IMPORTED_LOCATION ${_cares})
-      if (cares_INCLUDE_DIRS)
-        set_target_properties (c-ares::cares PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${cares_INCLUDE_DIRS})
-      endif ()
+      add_library (c-ares::cares ${_static} IMPORTED GLOBAL)
+      set_target_properties (c-ares::cares PROPERTIES
+        IMPORTED_LOCATION ${_location}
+        INTERFACE_INCLUDE_DIRECTORIES "${${_prefix}_INCLUDE_DIRS}"
+        INTERFACE_LINK_OPTIONS "${${_prefix}_LDFLAGS}"
+      )
       exclude_if_included (c-ares::cares)
     else ()
       message (FATAL_ERROR "using pkg-config for grpc, can't find c-ares")

cfg/rippled-example.cfg

@@ -593,6 +593,23 @@
 #
 #
 #
+#
+# [relay_proposals]
+#
+#   Controls the relaying behavior for proposals received by this server that
+#   are issued by validators that are not on the server's UNL.
+#
+#   Legal values are: "trusted" and "all". The default is "trusted".
+#
+#
+# [relay_validations]
+#
+#   Controls the relaying behavior for validations received by this server that
+#   are issued by validators that are not on the server's UNL.
+#
+#   Legal values are: "trusted" and "all". The default is "all".
+#
+#
 # [node_size]
 #
 #   Tunes the servers based on the expected load and available memory. Legal

src/ripple/app/consensus/RCLConsensus.cpp

@@ -217,7 +217,7 @@ RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal)
 
     app_.getHashRouter().addSuppression(suppression);
 
-    app_.overlay().send(prop);
+    app_.overlay().broadcast(prop);
 }
 
 void
@@ -828,12 +828,17 @@ RCLConsensus::Adaptor::validate(
     // suppress it if we receive it
     app_.getHashRouter().addSuppression(
         sha512Half(makeSlice(v->getSerialized())));
+
     handleNewValidation(app_, v, "local");
+
+    // Broadcast to all our peers:
     Blob validation = v->getSerialized();
     protocol::TMValidation val;
     val.set_validation(&validation[0], validation.size());
-    // Send signed validation to all of our directly connected peers
-    app_.overlay().send(val);
+    app_.overlay().broadcast(val);
+
+    // Publish to all our subscribers:
+    app_.getOPs().pubValidation(v);
 }
 
 void

src/ripple/app/consensus/RCLValidations.cpp

@@ -148,7 +148,7 @@ RCLValidationsAdaptor::acquire(LedgerHash const& hash)
     return RCLValidatedLedger(std::move(ledger), j_);
 }
 
-bool
+void
 handleNewValidation(
     Application& app,
     std::shared_ptr<STValidation> const& val,
@@ -166,7 +166,6 @@ handleNewValidation(
     if (!masterKey)
         masterKey = app.validators().getListedKey(signingKey);
 
-    bool shouldRelay = false;
     RCLValidations& validations = app.getValidations();
     beast::Journal const j = validations.adaptor().journal();
 
@@ -182,67 +181,42 @@ handleNewValidation(
           << " [" << val->getSerializer().slice() << "]";
     };
 
-    if (!val->isFieldPresent(sfLedgerSequence))
-    {
-        if (j.error())
-            dmp(j.error(), "missing ledger sequence field");
-        return false;
-    }
-
     // masterKey is seated only if validator is trusted or listed
     if (masterKey)
     {
         ValStatus const outcome = validations.add(calcNodeID(*masterKey), val);
+        auto const seq = val->getFieldU32(sfLedgerSequence);
 
         if (j.debug())
             dmp(j.debug(), to_string(outcome));
 
+        // One might think that we would not wish to relay validations that
+        // fail these checks. Somewhat counterintuitively, we actually want
+        // to do it for validations that we receive but deem suspicious, so
+        // that our peers will also observe them and realize they're bad.
         if (outcome == ValStatus::conflicting && j.warn())
         {
-            auto const seq = val->getFieldU32(sfLedgerSequence);
             dmp(j.warn(),
                 "conflicting validations issued for " + to_string(seq) +
                     " (likely from a Byzantine validator)");
         }
 
         if (outcome == ValStatus::multiple && j.warn())
         {
-            auto const seq = val->getFieldU32(sfLedgerSequence);
             dmp(j.warn(),
                 "multiple validations issued for " + to_string(seq) +
                     " (multiple validators operating with the same key?)");
         }
 
-        if (outcome == ValStatus::badSeq && j.warn())
-        {
-            auto const seq = val->getFieldU32(sfLedgerSequence);
-            dmp(j.debug(),
-                "already validated sequence at or past " + std::to_string(seq));
-        }
-
         if (val->isTrusted() && outcome == ValStatus::current)
-        {
-            app.getLedgerMaster().checkAccept(
-                hash, val->getFieldU32(sfLedgerSequence));
-            shouldRelay = true;
-        }
+            app.getLedgerMaster().checkAccept(hash, seq);
     }
     else
     {
         JLOG(j.debug()) << "Val for " << hash << " from "
                         << toBase58(TokenType::NodePublic, signingKey)
                         << " not added UNlisted";
     }
-
-    // This currently never forwards untrusted validations, though we may
-    // reconsider in the future. From @JoelKatz:
-    // The idea was that we would have a certain number of validation slots with
-    // priority going to validators we trusted. Remaining slots might be
-    // allocated to validators that were listed by publishers we trusted but
-    // that we didn't choose to trust. The shorter term plan was just to forward
-    // untrusted validations if peers wanted them or if we had the
-    // ability/bandwidth to. None of that was implemented.
-    return shouldRelay;
 }
 
 }  // namespace ripple

src/ripple/app/consensus/RCLValidations.h

@@ -243,10 +243,8 @@ using RCLValidations = Validations<RCLValidationsAdaptor>;
     @param app Application object containing validations and ledgerMaster
     @param val The validation to add
     @param source Name associated with validation used in logging
-
-    @return Whether the validation should be relayed
 */
-bool
+void
 handleNewValidation(
     Application& app,
     std::shared_ptr<STValidation> const& val,

src/ripple/app/ledger/Ledger.cpp

@@ -499,7 +499,6 @@ Ledger::rawInsert(std::shared_ptr<SLE> const& sle)
     Serializer ss;
     sle->add(ss);
     auto item = std::make_shared<SHAMapItem const>(sle->key(), std::move(ss));
-    // VFALCO NOTE addGiveItem should take ownership
     if (!stateMap_->addGiveItem(std::move(item), false, false))
         LogicError("Ledger::rawInsert: key already exists");
 }
@@ -510,7 +509,7 @@ Ledger::rawReplace(std::shared_ptr<SLE> const& sle)
     Serializer ss;
     sle->add(ss);
     auto item = std::make_shared<SHAMapItem const>(sle->key(), std::move(ss));
-    // VFALCO NOTE updateGiveItem should take ownership
+
     if (!stateMap_->updateGiveItem(std::move(item), false, false))
         LogicError("Ledger::rawReplace: key not found");
 }

src/ripple/app/ledger/LedgerMaster.h

@@ -286,6 +286,10 @@ class LedgerMaster : public Stoppable, public AbstractFetchPackContainer
         return !mValidLedger.empty();
     }
 
+    // Returns the minimum ledger sequence in SQL database, if any.
+    boost::optional<LedgerIndex>
+    minSqlSeq();
+
 private:
     void
     setValidLedger(std::shared_ptr<Ledger const> const& l);

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -143,29 +143,17 @@ static constexpr std::chrono::minutes MAX_LEDGER_AGE_ACQUIRE{1};
 // Don't acquire history if write load is too high
 static constexpr int MAX_WRITE_LOAD_ACQUIRE{8192};
 
-// Helper function for LedgerMaster::doAdvance()
-// Returns the minimum ledger sequence in SQL database, if any.
-static boost::optional<LedgerIndex>
-minSqlSeq(Application& app)
-{
-    boost::optional<LedgerIndex> seq;
-    auto db = app.getLedgerDB().checkoutDb();
-    *db << "SELECT MIN(LedgerSeq) FROM Ledgers", soci::into(seq);
-    return seq;
-}
-
 // Helper function for LedgerMaster::doAdvance()
 // Return true if candidateLedger should be fetched from the network.
 static bool
 shouldAcquire(
     std::uint32_t const currentLedger,
     std::uint32_t const ledgerHistory,
-    boost::optional<LedgerIndex> minSeq,
-    std::uint32_t const lastRotated,
+    boost::optional<LedgerIndex> const minimumOnline,
     std::uint32_t const candidateLedger,
     beast::Journal j)
 {
-    bool ret = [&]() {
+    bool const ret = [&]() {
         // Fetch ledger if it may be the current ledger
         if (candidateLedger >= currentLedger)
             return true;
@@ -174,15 +162,9 @@ shouldAcquire(
         if (currentLedger - candidateLedger <= ledgerHistory)
             return true;
 
-        // Or it's greater than or equal to both:
-        // - the minimum persisted ledger or the maximum possible
-        //   sequence value, if no persisted ledger, and
-        // - minimum ledger that will be persisted as of the next online
-        //   deletion interval, or 1 if online deletion is disabled.
-        return candidateLedger >=
-            std::max(
-                   minSeq.value_or(std::numeric_limits<LedgerIndex>::max()),
-                   lastRotated + 1);
+        // Or if greater than or equal to a specific minimum ledger.
+        // Do nothing if the minimum ledger to keep online is unknown.
+        return minimumOnline.has_value() && candidateLedger >= *minimumOnline;
     }();
 
     JLOG(j.trace()) << "Missing ledger " << candidateLedger
@@ -1899,8 +1881,7 @@ LedgerMaster::doAdvance(std::unique_lock<std::recursive_mutex>& sl)
                         shouldAcquire(
                             mValidLedgerSeq,
                             ledger_history_,
-                            minSqlSeq(app_),
-                            app_.getSHAMapStore().getLastRotated(),
+                            app_.getSHAMapStore().minimumOnline(),
                             *missing,
                             m_journal))
                     {
@@ -2149,4 +2130,14 @@ LedgerMaster::getFetchPackCacheSize() const
     return fetch_packs_.getCacheSize();
 }
 
+// Returns the minimum ledger sequence in SQL database, if any.
+boost::optional<LedgerIndex>
+LedgerMaster::minSqlSeq()
+{
+    boost::optional<LedgerIndex> seq;
+    auto db = app_.getLedgerDB().checkoutDb();
+    *db << "SELECT MIN(LedgerSeq) FROM Ledgers", soci::into(seq);
+    return seq;
+}
+
 }  // namespace ripple

src/ripple/app/ledger/impl/LedgerReplay.cpp

@@ -29,7 +29,7 @@ LedgerReplay::LedgerReplay(
 {
     for (auto const& item : replay_->txMap())
     {
-        auto const txPair = replay_->txRead(item.key());
+        auto txPair = replay_->txRead(item.key());  // non-const so can be moved
         auto const txIndex = (*txPair.second)[sfTransactionIndex];
         orderedTxns_.emplace(txIndex, std::move(txPair.first));
     }