diff --git a/src/ripple_app/data/DatabaseCon.cpp b/src/ripple_app/data/DatabaseCon.cpp index 5d03b0d283c..2fe4efe07f2 100644 --- a/src/ripple_app/data/DatabaseCon.cpp +++ b/src/ripple_app/data/DatabaseCon.cpp @@ -25,10 +25,14 @@ DatabaseCon::DatabaseCon (const std::string& strName, const char* initStrings[], // responsibility to pass in the path. Add a member function to Application // or Config to compute this path. // - boost::filesystem::path pPath = (getConfig ().RUN_STANDALONE && - ((getConfig ().START_UP != Config::LOAD) && (getConfig ().START_UP != Config::REPLAY))) - ? "" // Use temporary files. - : (getConfig ().DATA_DIR / strName); // Use regular db files. + auto const startUp = getConfig ().START_UP; + auto const useTempFiles // Use temporary files or regular DB files? + = getConfig ().RUN_STANDALONE && + startUp != Config::LOAD && + startUp != Config::LOAD_FILE && + startUp != Config::REPLAY; + boost::filesystem::path pPath = useTempFiles + ? "" : (getConfig ().DATA_DIR / strName); mDatabase = new SqliteDatabase (pPath.string ().c_str ()); mDatabase->connect (); diff --git a/src/ripple_app/ledger/Ledger.cpp b/src/ripple_app/ledger/Ledger.cpp index 29e2b9cd850..f87c7cdd396 100644 --- a/src/ripple_app/ledger/Ledger.cpp +++ b/src/ripple_app/ledger/Ledger.cpp @@ -204,6 +204,28 @@ Ledger::Ledger (const std::string& rawLedger, bool hasPrefix) initializeFees (); } +/** Used for ledgers loaded from JSON files */ +Ledger::Ledger (std::uint32_t ledgerSeq, std::uint32_t closeTime) + : mTotCoins (0), + mLedgerSeq (ledgerSeq), + mCloseTime (closeTime), + mParentCloseTime (0), + mCloseResolution (LEDGER_TIME_ACCURACY), + mCloseFlags (0), + mClosed (false), + mValidated (false), + mValidHash (false), + mAccepted (false), + mImmutable (false), + mTransactionMap (boost::make_shared ( + smtTRANSACTION, std::ref (getApp().getFullBelowCache()))), + mAccountStateMap (boost::make_shared ( + smtSTATE, std::ref (getApp().getFullBelowCache()))) +{ + initializeFees (); +} + + Ledger::~Ledger () { if (mTransactionMap) @@ -326,6 +348,12 @@ bool Ledger::hasAccount (const RippleAddress& accountID) return mAccountStateMap->hasItem (Ledger::getAccountRootIndex (accountID)); } +bool Ledger::addSLE (SLE const& sle) +{ + SHAMapItem item (sle.getIndex(), sle.getSerializer()); + return mAccountStateMap->addItem(item, false, false); +} + AccountState::pointer Ledger::getAccountState (const RippleAddress& accountID) { #ifdef BEAST_DEBUG diff --git a/src/ripple_app/ledger/Ledger.h b/src/ripple_app/ledger/Ledger.h index 86f3d52ebb2..671dca57b9c 100644 --- a/src/ripple_app/ledger/Ledger.h +++ b/src/ripple_app/ledger/Ledger.h @@ -109,6 +109,8 @@ class Ledger int closeFlags, int closeResolution, std::uint32_t ledgerSeq, bool & loaded); // used for database ledgers + Ledger (std::uint32_t ledgerSeq, std::uint32_t closeTime); + Ledger (Blob const & rawLedger, bool hasPrefix); Ledger (const std::string & rawLedger, bool hasPrefix); @@ -188,6 +190,10 @@ class Ledger { mTotCoins -= fee; } + void setTotalCoins (std::uint64_t totCoins) + { + mTotCoins = totCoins; + } std::uint32_t getCloseTimeNC () const { return mCloseTime; @@ -238,6 +244,9 @@ class Ledger mAccountStateMap->dropCache (); } + // returns false on error + bool addSLE (SLE const& sle); + // ledger sync functions void setAcquiring (void); bool isAcquiring (void); diff --git a/src/ripple_app/ledger/LedgerTiming.h b/src/ripple_app/ledger/LedgerTiming.h index e4b86783144..94b88d78294 100644 --- a/src/ripple_app/ledger/LedgerTiming.h +++ b/src/ripple_app/ledger/LedgerTiming.h @@ -23,58 +23,65 @@ namespace ripple { // The number of seconds a ledger may remain idle before closing -# define LEDGER_IDLE_INTERVAL 15 +const int LEDGER_IDLE_INTERVAL = 15; // The number of seconds a validation remains current after its ledger's close time // This is a safety to protect against very old validations and the time it takes to adjust // the close time accuracy window -# define LEDGER_VAL_INTERVAL 300 +const int LEDGER_VAL_INTERVAL = 300; // The number of seconds before a close time that we consider a validation acceptable // This protects against extreme clock errors -# define LEDGER_EARLY_INTERVAL 180 +const int LEDGER_EARLY_INTERVAL = 180; // The number of milliseconds we wait minimum to ensure participation -# define LEDGER_MIN_CONSENSUS 2000 +const int LEDGER_MIN_CONSENSUS = 2000; // The number of milliseconds we wait minimum to ensure others have computed the LCL -# define LEDGER_MIN_CLOSE 2000 +const int LEDGER_MIN_CLOSE = 2000; // Initial resolution of ledger close time -# define LEDGER_TIME_ACCURACY 30 +const int LEDGER_TIME_ACCURACY = 30; // How often to increase resolution -# define LEDGER_RES_INCREASE 8 +const int LEDGER_RES_INCREASE = 8; // How often to decrease resolution -# define LEDGER_RES_DECREASE 1 +const int LEDGER_RES_DECREASE = 1; // How often we check state or change positions (in milliseconds) -# define LEDGER_GRANULARITY 1000 +const int LEDGER_GRANULARITY = 1000; // The percentage of active trusted validators that must be able to // keep up with the network or we consider the network overloaded -# define LEDGER_NET_RATIO 70 +const int LEDGER_NET_RATIO = 70; // How long we consider a proposal fresh -# define PROPOSE_FRESHNESS 20 +const int PROPOSE_FRESHNESS = 20; // How often we force generating a new proposal to keep ours fresh -# define PROPOSE_INTERVAL 12 +const int PROPOSE_INTERVAL = 12; // Avalanche tuning -# define AV_INIT_CONSENSUS_PCT 50 // percentage of nodes on our UNL that must vote yes +// percentage of nodes on our UNL that must vote yes +const int AV_INIT_CONSENSUS_PCT = 50; -# define AV_MID_CONSENSUS_TIME 50 // percentage of previous close time before we advance -# define AV_MID_CONSENSUS_PCT 65 // percentage of nodes that most vote yes after advancing +// percentage of previous close time before we advance +const int AV_MID_CONSENSUS_TIME = 50; -# define AV_LATE_CONSENSUS_TIME 85 // percentage of previous close time before we advance -# define AV_LATE_CONSENSUS_PCT 70 // percentage of nodes that most vote yes after advancing +// percentage of nodes that most vote yes after advancing +const int AV_MID_CONSENSUS_PCT = 65; -# define AV_STUCK_CONSENSUS_TIME 200 -# define AV_STUCK_CONSENSUS_PCT 95 +// percentage of previous close time before we advance +const int AV_LATE_CONSENSUS_TIME = 85; -# define AV_CT_CONSENSUS_PCT 75 +// percentage of nodes that most vote yes after advancing +const int AV_LATE_CONSENSUS_PCT = 70; + +const int AV_STUCK_CONSENSUS_TIME = 200; +const int AV_STUCK_CONSENSUS_PCT = 95; + +const int AV_CT_CONSENSUS_PCT = 75; class ContinuousLedgerTiming { diff --git a/src/ripple_app/main/Application.cpp b/src/ripple_app/main/Application.cpp index 354ca7aae32..cd68660cc8b 100644 --- a/src/ripple_app/main/Application.cpp +++ b/src/ripple_app/main/Application.cpp @@ -662,26 +662,31 @@ class ApplicationImp m_ledgerMaster->setMinValidations (getConfig ().VALIDATION_QUORUM); - if (getConfig ().START_UP == Config::FRESH) + auto const startUp = getConfig ().START_UP; + if (startUp == Config::FRESH) { m_journal.info << "Starting new Ledger"; startNewLedger (); } - else if ((getConfig ().START_UP == Config::LOAD) || (getConfig ().START_UP == Config::REPLAY)) + else if (startUp == Config::LOAD || + startUp == Config::LOAD_FILE || + startUp == Config::REPLAY) { m_journal.info << "Loading specified Ledger"; - if (!loadOldLedger (getConfig ().START_LEDGER, getConfig ().START_UP == Config::REPLAY)) + if (!loadOldLedger (getConfig ().START_LEDGER, + startUp == Config::REPLAY, + startUp == Config::LOAD_FILE)) { // wtf? getApp().signalStop (); exit (-1); } } - else if (getConfig ().START_UP == Config::NETWORK) + else if (startUp == Config::NETWORK) { - // This should probably become the default once we have a stable network + // This should probably become the default once we have a stable network. if (!getConfig ().RUN_STANDALONE) m_networkOPs->needNetworkLedger (); @@ -692,9 +697,10 @@ class ApplicationImp m_orderBookDB.setup (getApp().getLedgerMaster ().getCurrentLedger ()); - // // Begin validation and ip maintenance. - // - LocalCredentials maintains local information: including identity and network connection persistence information. + // + // - LocalCredentials maintains local information: including identity + // - and network connection persistence information. // // VFALCO NOTE this starts the UNL m_localCredentials.start (); @@ -1095,7 +1101,8 @@ class ApplicationImp private: void updateTables (); void startNewLedger (); - bool loadOldLedger (const std::string&, bool); + bool loadOldLedger ( + const std::string& ledgerID, bool replay, bool isFilename); void onAnnounceAddress (); }; @@ -1116,8 +1123,8 @@ void ApplicationImp::startNewLedger () { Ledger::pointer firstLedger = boost::make_shared (rootAddress, SYSTEM_CURRENCY_START); assert (!!firstLedger->getAccountState (rootAddress)); - // WRITEME: Add any default amendments - // WRITEME: Set default fee/reserve + // TODO(david): Add any default amendments + // TODO(david): Set default fee/reserve firstLedger->updateHash (); firstLedger->setClosed (); firstLedger->setAccepted (); @@ -1132,34 +1139,125 @@ void ApplicationImp::startNewLedger () } } -bool ApplicationImp::loadOldLedger (const std::string& l, bool bReplay) +bool ApplicationImp::loadOldLedger ( + const std::string& ledgerID, bool replay, bool isFileName) { try { Ledger::pointer loadLedger, replayLedger; - if (l.empty () || (l == "latest")) + if (isFileName) + { + std::ifstream ledgerFile (ledgerID.c_str (), std::ios::in); + if (!ledgerFile) + { + m_journal.fatal << "Unable to open file"; + } + else + { + Json::Reader reader; + Json::Value jLedger; + if (!reader.parse (ledgerFile, jLedger, false)) + m_journal.fatal << "Unable to parse ledger JSON"; + else + { + std::reference_wrapper ledger (jLedger); + + // accept a wrapped ledger + if (ledger.get().isMember ("result")) + ledger = ledger.get()["result"]; + if (ledger.get().isMember ("ledger")) + ledger = ledger.get()["ledger"]; + + + std::uint32_t seq = 1; + std::uint32_t closeTime = getApp().getOPs().getCloseTimeNC (); + std::uint64_t totalCoins = 0; + + if (ledger.get().isMember ("accountState")) + { + if (ledger.get().isMember ("ledger_index")) + { + seq = ledger.get()["ledger_index"].asUInt(); + } + if (ledger.get().isMember ("close_time")) + { + closeTime = ledger.get()["close_time"].asUInt(); + } + if (ledger.get().isMember ("total_coins")) + { + totalCoins = + beast::lexicalCastThrow + (ledger.get()["total_coins"].asString()); + } + ledger = ledger.get()["accountState"]; + } + if (!ledger.get().isArray ()) + { + m_journal.fatal << "State nodes must be an array"; + } + else + { + loadLedger = boost::make_shared (seq, closeTime); + loadLedger->setTotalCoins(totalCoins); + + for (Json::UInt index = 0; index < ledger.get().size(); ++index) + { + Json::Value& entry = ledger.get()[index]; + + uint256 uIndex; + uIndex.SetHex (entry["index"].asString()); + entry.removeMember ("index"); + + STParsedJSON stp ("sle", ledger.get()[index]); + // m_journal.info << "json: " << stp.object->getJson(0); + + if (stp.object && (uIndex.isNonZero())) + { + SerializedLedgerEntry sle (*stp.object, uIndex); + bool ok = loadLedger->addSLE (sle); + if (!ok) + m_journal.warning << "Couldn't add serialized ledger: " << uIndex; + } + else + { + m_journal.warning << "Invalid entry in ledger"; + } + } + // TODO(david): close ledger, update hash + } + } + } + } + else if (ledgerID.empty () || (ledgerID == "latest")) loadLedger = Ledger::getLastFullLedger (); - else if (l.length () == 64) + else if (ledgerID.length () == 64) { // by hash uint256 hash; - hash.SetHex (l); + hash.SetHex (ledgerID); loadLedger = Ledger::loadByHash (hash); } else // assume by sequence - loadLedger = Ledger::loadByIndex (beast::lexicalCastThrow (l)); + loadLedger = Ledger::loadByIndex ( + beast::lexicalCastThrow (ledgerID)); if (!loadLedger) { - m_journal.fatal << "No Ledger found?" << std::endl; + m_journal.fatal << "No Ledger found from ledgerID=" + << ledgerID << std::endl; return false; } - if (bReplay) - { // Replay a ledger close with same prior ledger and transactions - replayLedger = loadLedger; // this ledger holds the transactions we want to replay - loadLedger = Ledger::loadByIndex (replayLedger->getLedgerSeq() - 1); // this is the prior ledger + if (replay) + { + // Replay a ledger close with same prior ledger and transactions + + // this ledger holds the transactions we want to replay + replayLedger = loadLedger; + + // this is the prior ledger + loadLedger = Ledger::loadByIndex (replayLedger->getLedgerSeq() - 1); if (!loadLedger || (replayLedger->getParentHash() != loadLedger->getHash())) { m_journal.fatal << "Replay ledger missing/damaged"; @@ -1182,12 +1280,14 @@ bool ApplicationImp::loadOldLedger (const std::string& l, bool bReplay) if (!loadLedger->walkLedger ()) { m_journal.fatal << "Ledger is missing nodes."; + assert(false); return false; } if (!loadLedger->assertSane ()) { m_journal.fatal << "Ledger is not sane."; + assert(false); return false; } @@ -1198,21 +1298,21 @@ bool ApplicationImp::loadOldLedger (const std::string& l, bool bReplay) m_ledgerMaster->forceValid(loadLedger); m_networkOPs->setLastCloseTime (loadLedger->getCloseTimeNC ()); - if (bReplay) - { // inject transaction from replayLedger into consensus set + if (replay) + { + // inject transaction from replayLedger into consensus set SHAMap::ref txns = replayLedger->peekTransactionMap(); Ledger::ref cur = getLedgerMaster().getCurrentLedger(); - for (SHAMapItem::pointer it = txns->peekFirstItem(); it != nullptr; it = txns->peekNextItem(it->getTag())) + for (auto it = txns->peekFirstItem(); it != nullptr; + it = txns->peekNextItem(it->getTag())) { Transaction::pointer txn = replayLedger->getTransaction(it->getTag()); m_journal.info << txn->getJson(0); Serializer s; txn->getSTransaction()->add(s); if (!cur->addTransaction(it->getTag(), s)) - { m_journal.warning << "Unable to add transaction " << it->getTag(); - } } } } @@ -1223,7 +1323,7 @@ bool ApplicationImp::loadOldLedger (const std::string& l, bool bReplay) } catch (boost::bad_lexical_cast&) { - m_journal.fatal << "Ledger specified '" << l << "' is not valid"; + m_journal.fatal << "Ledger specified '" << ledgerID << "' is not valid"; return false; } diff --git a/src/ripple_app/main/Main.cpp b/src/ripple_app/main/Main.cpp index 2b60e66e994..cff234a1c52 100644 --- a/src/ripple_app/main/Main.cpp +++ b/src/ripple_app/main/Main.cpp @@ -204,6 +204,7 @@ int run (int argc, char** argv) ("load", "Load the current ledger from the local DB.") ("replay","Replay a ledger close.") ("ledger", po::value (), "Load the specified ledger and start from .") + ("ledgerfile", po::value (), "Load the specified ledger file.") ("start", "Start from a fresh Ledger.") ("net", "Get the initial ledger from the network.") ("fg", "Run in the foreground.") @@ -330,6 +331,11 @@ int run (int argc, char** argv) else getConfig ().START_UP = Config::LOAD; } + else if (vm.count ("ledgerfile")) + { + getConfig ().START_LEDGER = vm["ledgerfile"].as (); + getConfig ().START_UP = Config::LOAD_FILE; + } else if (vm.count ("load")) { getConfig ().START_UP = Config::LOAD; diff --git a/src/ripple_app/misc/SerializedLedger.cpp b/src/ripple_app/misc/SerializedLedger.cpp index e0be0826bb4..c784b0c3ed9 100644 --- a/src/ripple_app/misc/SerializedLedger.cpp +++ b/src/ripple_app/misc/SerializedLedger.cpp @@ -23,43 +23,44 @@ struct SerializedLedgerLog; // for Log SETUP_LOGN (SerializedLedgerLog,"SerializedLedger") -SerializedLedgerEntry::SerializedLedgerEntry (SerializerIterator& sit, uint256 const& index) +SerializedLedgerEntry::SerializedLedgerEntry ( + SerializerIterator& sit, uint256 const& index) : STObject (sfLedgerEntry), mIndex (index), mMutable (true) { set (sit); - std::uint16_t type = getFieldU16 (sfLedgerEntryType); - - LedgerFormats::Item const* const item = - LedgerFormats::getInstance()->findByType (static_cast (type)); - - if (item == nullptr) - throw std::runtime_error ("invalid ledger entry type"); - - mType = item->getType (); - - if (!setType (item->elements)) - throw std::runtime_error ("ledger entry not valid for type"); + setSLEType (); } -SerializedLedgerEntry::SerializedLedgerEntry (const Serializer& s, uint256 const& index) +SerializedLedgerEntry::SerializedLedgerEntry ( + const Serializer& s, uint256 const& index) : STObject (sfLedgerEntry), mIndex (index), mMutable (true) { - SerializerIterator sit (const_cast (s)); // we know 's' isn't going away + // we know 's' isn't going away + SerializerIterator sit (const_cast (s)); set (sit); + setSLEType (); +} - std::uint16_t type = getFieldU16 (sfLedgerEntryType); +SerializedLedgerEntry::SerializedLedgerEntry ( + const STObject & object, uint256 const& index) + : STObject (object), mIndex(index), mMutable (true) +{ + setSLEType (); +} - LedgerFormats::Item const* const item = - LedgerFormats::getInstance()->findByType (static_cast (type)); +void SerializedLedgerEntry::setSLEType () +{ + auto type = static_cast (getFieldU16 (sfLedgerEntryType)); + auto const item = LedgerFormats::getInstance()->findByType (type); if (item == nullptr) throw std::runtime_error ("invalid ledger entry type"); mType = item->getType (); - if (!setType (item->elements)) { - WriteLog (lsWARNING, SerializedLedgerLog) << "Ledger entry not valid for type " << mFormat->getName (); + WriteLog (lsWARNING, SerializedLedgerLog) + << "Ledger entry not valid for type " << mFormat->getName (); WriteLog (lsWARNING, SerializedLedgerLog) << getJson (0); throw std::runtime_error ("ledger entry not valid for type"); } diff --git a/src/ripple_app/misc/SerializedLedger.h b/src/ripple_app/misc/SerializedLedger.h index 58f54990c07..6c8cc30dc8d 100644 --- a/src/ripple_app/misc/SerializedLedger.h +++ b/src/ripple_app/misc/SerializedLedger.h @@ -51,6 +51,7 @@ class SerializedLedgerEntry SerializedLedgerEntry (const Serializer & s, uint256 const & index); SerializedLedgerEntry (SerializerIterator & sit, uint256 const & index); SerializedLedgerEntry (LedgerEntryType type, uint256 const & index); + SerializedLedgerEntry (const STObject & object, uint256 const & index); SerializedTypeID getSType () const { @@ -111,6 +112,11 @@ class SerializedLedgerEntry return new SerializedLedgerEntry (*this); } + /** Make STObject comply with the template for this SLE type + Can throw + */ + void setSLEType (); + private: uint256 mIndex; LedgerEntryType mType; diff --git a/src/ripple_app/ripple_app.cpp b/src/ripple_app/ripple_app.cpp index fda7e88d63d..556153888da 100644 --- a/src/ripple_app/ripple_app.cpp +++ b/src/ripple_app/ripple_app.cpp @@ -46,6 +46,8 @@ #include "../beast/beast/asio/io_latency_probe.h" #include "../beast/beast/cxx14/memory.h" +#include // For Application.cpp + # include "main/CollectorManager.h" #include "main/CollectorManager.cpp" diff --git a/src/ripple_core/functional/Config.h b/src/ripple_core/functional/Config.h index 31964ebf5d7..fb8e16fb833 100644 --- a/src/ripple_core/functional/Config.h +++ b/src/ripple_core/functional/Config.h @@ -360,6 +360,7 @@ class Config FRESH, NORMAL, LOAD, + LOAD_FILE, REPLAY, NETWORK }; diff --git a/src/ripple_data/protocol/STParsedJSON.cpp b/src/ripple_data/protocol/STParsedJSON.cpp index b014354b6f1..22826dd975b 100644 --- a/src/ripple_data/protocol/STParsedJSON.cpp +++ b/src/ripple_data/protocol/STParsedJSON.cpp @@ -428,10 +428,10 @@ bool STParsedJSON::parse (std::string const& json_name, STVector256* tail (dynamic_cast (&data.back ())); assert (tail); - for (Json::UInt i = 0; !json.isValidIndex (i); ++i) + for (Json::UInt i = 0; value.isValidIndex (i); ++i) { uint256 s; - s.SetHex (json[i].asString ()); + s.SetHex (value[i].asString ()); tail->addValue (s); } }