Proposed 1.6.0-b7

.travis.yml

@@ -408,7 +408,8 @@ cache:
     - $CACHE_DIR
 
 before_install:
-  - if [ "$(uname)" = "Darwin" ] ; then export NUM_PROCESSORS=$(sysctl -n hw.physicalcpu); else export NUM_PROCESSORS=$(nproc); fi
+  # NUM_PROCESSORS was set to 1 due to problems in parallel launch of unit tests on Mac platform
+  - if [ "$(uname)" = "Darwin" ] ; then export NUM_PROCESSORS=1; else export NUM_PROCESSORS=$(nproc); fi
   - echo "NUM PROC is ${NUM_PROCESSORS}"
   - if [ "$(uname)" = "Linux" ] ; then docker pull ${DOCKER_IMAGE}; fi
   - if [ "${MATRIX_EVAL}" != "" ] ; then eval "${MATRIX_EVAL}"; fi

Builds/CMake/RippledCore.cmake

@@ -844,6 +844,7 @@ target_sources (rippled PRIVATE
   #]===============================]
   src/test/nodestore/Backend_test.cpp
   src/test/nodestore/Basics_test.cpp
+  src/test/nodestore/DatabaseShard_test.cpp
   src/test/nodestore/Database_test.cpp
   src/test/nodestore/Timing_test.cpp
   src/test/nodestore/import_test.cpp

Builds/CMake/deps/Findlibarchive_pc.cmake

@@ -1,5 +1,5 @@
 find_package (PkgConfig REQUIRED)
-pkg_search_module (libarchive_PC QUIET libarchive>=3.3.3)
+pkg_search_module (libarchive_PC QUIET libarchive>=3.4.3)
 
 if(static)
   set(LIBARCHIVE_LIB libarchive.a)

Builds/CMake/deps/Findlz4.cmake

@@ -1,6 +1,6 @@
 find_package (PkgConfig)
 if (PKG_CONFIG_FOUND)
-  pkg_search_module (lz4_PC QUIET liblz4>=1.8)
+  pkg_search_module (lz4_PC QUIET liblz4>=1.9)
 endif ()
 
 if(static)

Builds/CMake/deps/Libarchive.cmake

@@ -29,7 +29,7 @@ if (NOT local_libarchive)
       endif ()
   else ()
     ## now try searching using the minimal find module that cmake provides
-    find_package(LibArchive 3.3.3 QUIET)
+    find_package(LibArchive 3.4.3 QUIET)
     if (LibArchive_FOUND)
       if (static)
         # find module doesn't find static libs currently, so we re-search
@@ -70,7 +70,7 @@ if (local_libarchive)
   ExternalProject_Add (libarchive
     PREFIX ${nih_cache_path}
     GIT_REPOSITORY https://github.com/libarchive/libarchive.git
-    GIT_TAG v3.3.3
+    GIT_TAG v3.4.3
     CMAKE_ARGS
       # passing the compiler seems to be needed for windows CI, sadly
       -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}

Builds/CMake/deps/Lz4.cmake

@@ -21,7 +21,7 @@ else()
   ExternalProject_Add (lz4
     PREFIX ${nih_cache_path}
     GIT_REPOSITORY https://github.com/lz4/lz4.git
-    GIT_TAG v1.8.2
+    GIT_TAG v1.9.2
     SOURCE_SUBDIR contrib/cmake_unofficial
     CMAKE_ARGS
       -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}

Builds/containers/gitlab-ci/pkgbuild.yml

@@ -282,7 +282,7 @@ tag_bld_images:
   variables:
     docker_driver: overlay2
   image:
-    name: docker:latest
+    name: docker:19.03.8
   services:
     # workaround for TLS issues - consider going back
     # back to unversioned `dind` when issues are resolved

src/ripple/nodestore/impl/DatabaseShardImp.cpp

@@ -135,13 +135,12 @@ DatabaseShardImp::init()
                     std::make_unique<Shard>(app_, *this, shardIndex, j_)};
                 if (!shard->open(scheduler_, *ctx_))
                 {
-                    if (!shard->isLegacy())
-                        return false;
-
-                    // Remove legacy shard
+                    // Remove corrupted or legacy shard
                     shard->removeOnDestroy();
                     JLOG(j_.warn())
-                        << "shard " << shardIndex << " removed, legacy shard";
+                        << "shard " << shardIndex << " removed, "
+                        << (shard->isLegacy() ? "legacy" : "corrupted")
+                        << " shard";
                     continue;
                 }
 
@@ -276,11 +275,11 @@ DatabaseShardImp::prepareShard(std::uint32_t shardIndex)
     // is greater or equal to the current shard.
     auto seqCheck = [&](std::uint32_t seq) {
         // seq will be greater than zero if valid
-        if (seq > earliestLedgerSeq() && shardIndex >= seqToShardIndex(seq))
+        if (seq >= earliestLedgerSeq() && shardIndex >= seqToShardIndex(seq))
             return fail("has an invalid index");
         return true;
     };
-    if (!seqCheck(app_.getLedgerMaster().getValidLedgerIndex()) ||
+    if (!seqCheck(app_.getLedgerMaster().getValidLedgerIndex() + 1) ||
         !seqCheck(app_.getLedgerMaster().getCurrentLedgerIndex()))
     {
         return false;
@@ -1100,6 +1099,9 @@ DatabaseShardImp::initConfig(std::lock_guard<std::mutex>&)
         ledgersPerShard_ = get<std::uint32_t>(section, "ledgers_per_shard");
         if (ledgersPerShard_ == 0 || ledgersPerShard_ % 256 != 0)
             return fail("'ledgers_per_shard' must be a multiple of 256");
+
+        earliestShardIndex_ = seqToShardIndex(earliestLedgerSeq());
+        avgShardFileSz_ = ledgersPerShard_ * kilobytes(192);
     }
 
     // NuDB is the default and only supported permanent storage backend

src/ripple/nodestore/impl/DatabaseShardImp.h

@@ -234,7 +234,7 @@ class DatabaseShardImp : public DatabaseShard
     std::uint32_t ledgersPerShard_ = ledgersPerShardDefault;
 
     // The earliest shard index
-    std::uint32_t const earliestShardIndex_;
+    std::uint32_t earliestShardIndex_;
 
     // Average storage space required by a shard (in bytes)
     std::uint64_t avgShardFileSz_;

src/ripple/nodestore/impl/codec.h

@@ -51,13 +51,12 @@ lz4_decompress(void const* in, std::size_t in_size, BufferFactory&& bf)
         Throw<std::runtime_error>("lz4 decompress: n == 0");
     void* const out = bf(result.second);
     result.first = out;
-    if (LZ4_decompress_fast(
+    if (LZ4_decompress_safe(
             reinterpret_cast<char const*>(in) + n,
             reinterpret_cast<char*>(out),
-            result.second) +
-            n !=
-        in_size)
-        Throw<std::runtime_error>("lz4 decompress: LZ4_decompress_fast");
+            in_size - n,
+            result.second) != result.second)
+        Throw<std::runtime_error>("lz4 decompress: LZ4_decompress_safe");
     return result;
 }
 

src/ripple/overlay/impl/OverlayImpl.cpp

@@ -554,14 +554,9 @@ OverlayImpl::onPrepare()
             for (auto const& addr : addresses)
             {
                 if (addr.port() == 0)
-                {
-                    Throw<std::runtime_error>(
-                        "Port not specified for "
-                        "address:" +
-                        addr.to_string());
-                }
-
-                ips.push_back(to_string(addr));
+                    ips.push_back(to_string(addr.at_port(DEFAULT_PEER_PORT)));
+                else
+                    ips.push_back(to_string(addr));
             }
 
             std::string const base("config: ");
@@ -577,8 +572,19 @@ OverlayImpl::onPrepare()
             [this](
                 std::string const& name,
                 std::vector<beast::IP::Endpoint> const& addresses) {
-                if (!addresses.empty())
-                    m_peerFinder->addFixedPeer(name, addresses);
+                std::vector<beast::IP::Endpoint> ips;
+                ips.reserve(addresses.size());
+
+                for (auto& addr : addresses)
+                {
+                    if (addr.port() == 0)
+                        ips.emplace_back(addr.address(), DEFAULT_PEER_PORT);
+                    else
+                        ips.emplace_back(addr);
+                }
+
+                if (!ips.empty())
+                    m_peerFinder->addFixedPeer(name, ips);
             });
     }
 }
@@ -1075,11 +1081,98 @@ OverlayImpl::processValidatorList(
     return true;
 }
 
+bool
+OverlayImpl::processHealth(http_request_type const& req, Handoff& handoff)
+{
+    if (req.target() != "/health")
+        return false;
+    boost::beast::http::response<json_body> msg;
+    msg.version(req.version());
+    msg.insert("Server", BuildInfo::getFullVersionString());
+    msg.insert("Content-Type", "application/json");
+    msg.insert("Connection", "close");
+
+    auto info = getServerInfo();
+
+    int last_validated_ledger_age = std::numeric_limits<int>::max();
+    if (info.isMember("validated_ledger"))
+        last_validated_ledger_age = info["validated_ledger"]["age"].asInt();
+    bool amendment_blocked = false;
+    if (info.isMember("amendment_blocked"))
+        amendment_blocked = true;
+    int number_peers = info["peers"].asInt();
+    std::string server_state = info["server_state"].asString();
+    auto load_factor = info["load_factor"].asDouble();
+
+    enum { healthy, warning, critical };
+    int health = healthy;
+    auto set_health = [&health](int state) {
+        if (health < state)
+            health = state;
+    };
+
+    if (last_validated_ledger_age >= 7)
+    {
+        msg.body()[jss::info]["validated_ledger"] = last_validated_ledger_age;
+        if (last_validated_ledger_age < 20)
+            set_health(warning);
+        else
+            set_health(critical);
+    }
+
+    if (amendment_blocked)
+    {
+        msg.body()[jss::info]["amendment_blocked"] = true;
+        set_health(critical);
+    }
+
+    if (number_peers <= 7)
+    {
+        msg.body()[jss::info]["peers"] = number_peers;
+        if (number_peers != 0)
+            set_health(warning);
+        else
+            set_health(critical);
+    }
+
+    if (!(server_state == "full" || server_state == "validating" ||
+          server_state == "proposing"))
+    {
+        msg.body()[jss::info]["server_state"] = server_state;
+        if (server_state == "syncing" || server_state == "tracking" ||
+            server_state == "connected")
+        {
+            set_health(warning);
+        }
+        else
+            set_health(critical);
+    }
+
+    if (load_factor > 100)
+    {
+        msg.body()[jss::info]["load_factor"] = load_factor;
+        if (load_factor < 1000)
+            set_health(warning);
+        else
+            set_health(critical);
+    }
+
+    if (health != critical)
+        msg.result(boost::beast::http::status::ok);
+    else
+        msg.result(boost::beast::http::status::service_unavailable);
+
+    msg.prepare_payload();
+    handoff.response = std::make_shared<SimpleWriter>(msg);
+    return true;
+}
+
 bool
 OverlayImpl::processRequest(http_request_type const& req, Handoff& handoff)
 {
     // Take advantage of || short-circuiting
-    return processCrawl(req, handoff) || processValidatorList(req, handoff);
+    return processCrawl(req, handoff) || processValidatorList(req, handoff) ||
+        processHealth(req, handoff);
 }
 
 Overlay::PeerSequence

src/ripple/overlay/impl/OverlayImpl.h

@@ -396,6 +396,14 @@ class OverlayImpl : public Overlay
     bool
     processValidatorList(http_request_type const& req, Handoff& handoff);
 
+    /** Handles health requests. Health returns information about the
+        health of the node.
+
+        @return true if the request was handled.
+    */
+    bool
+    processHealth(http_request_type const& req, Handoff& handoff);
+
     /** Handles non-peer protocol requests.
 
         @return true if the request was handled.

src/ripple/protocol/SystemParameters.h

@@ -61,4 +61,7 @@ static std::uint32_t constexpr XRP_LEDGER_EARLIEST_SEQ{32570};
 
 }  // namespace ripple
 
+/** Default peer port (IANA registered) */
+inline std::uint16_t constexpr DEFAULT_PEER_PORT{2459};
+
 #endif

src/ripple/protocol/impl/BuildInfo.cpp

@@ -33,7 +33,7 @@ namespace BuildInfo {
 //  and follow the format described at http://semver.org/
 //------------------------------------------------------------------------------
 // clang-format off
-char const* const versionString = "1.6.0-b6"
+char const* const versionString = "1.6.0-b7"
 // clang-format on
 
 #if defined(DEBUG) || defined(SANITIZER)

src/ripple/rpc/handlers/Connect.cpp

@@ -22,6 +22,7 @@
 #include <ripple/net/RPCErr.h>
 #include <ripple/overlay/Overlay.h>
 #include <ripple/protocol/ErrorCodes.h>
+#include <ripple/protocol/SystemParameters.h>
 #include <ripple/protocol/jss.h>
 #include <ripple/rpc/Context.h>
 #include <ripple/rpc/impl/Handler.h>
@@ -53,7 +54,7 @@ doConnect(RPC::JsonContext& context)
     if (context.params.isMember(jss::port))
         iPort = context.params[jss::port].asInt();
     else
-        iPort = 6561;
+        iPort = DEFAULT_PEER_PORT;
 
     auto ip =
         beast::IP::Endpoint::from_string(context.params[jss::ip].asString());

src/test/app/AccountTxPaging_test.cpp

@@ -1871,9 +1871,7 @@ class AccountTxPaging_test : public beast::unit_test::suite
         if (!BEAST_EXPECT(status.error_code() == 0))
             return;
 
-        if (!BEAST_EXPECT(
-                res.transactions().size() ==
-                std::extent<decltype(txCheck)>::value))
+        if (!BEAST_EXPECT(res.transactions().size() == std::size(txCheck)))
             return;
         for (int i = 0; i < res.transactions().size(); ++i)
         {

src/test/core/SociDB_test.cpp

@@ -321,11 +321,8 @@ class SociDB_test final : public TestSuite
                 LedgerSeq       BIGINT UNSIGNED             \
             );",
                 "CREATE INDEX SeqLedger ON Ledgers(LedgerSeq);"};
-            int dbInitCount = std::extent<decltype(dbInit)>::value;
-            for (int i = 0; i < dbInitCount; ++i)
-            {
-                s << dbInit[i];
-            }
+            for (auto const c : dbInit)
+                s << c;
             char lh[65];
             memset(lh, 'a', 64);
             lh[64] = '\0';

src/test/jtx/Env.h

@@ -325,8 +325,10 @@ class Env
 
             The Application network time is set to
             the close time of the resulting ledger.
+
+        @return true if no error, false if error
     */
-    void
+    bool
     close(
         NetClock::time_point closeTime,
         boost::optional<std::chrono::milliseconds> consensusDelay =
@@ -336,25 +338,29 @@ class Env
 
         The time is calculated as the duration from
         the previous ledger closing time.
+
+        @return true if no error, false if error
     */
     template <class Rep, class Period>
-    void
+    bool
     close(std::chrono::duration<Rep, Period> const& elapsed)
     {
         // VFALCO Is this the correct time?
-        close(now() + elapsed);
+        return close(now() + elapsed);
     }
 
     /** Close and advance the ledger.
 
         The time is calculated as five seconds from
         the previous ledger closing time.
+
+        @return true if no error, false if error
     */
-    void
+    bool
     close()
     {
         // VFALCO Is this the correct time?
-        close(std::chrono::seconds(5));
+        return close(std::chrono::seconds(5));
     }
 
     /** Turn on JSON tracing.