scraper: Add check for minimum housekeeping complete in scraper

src/gridcoin/quorum.cpp

@@ -23,7 +23,8 @@ unsigned int NumScrapersForSupermajority(unsigned int nScraperCount);
 mmCSManifestsBinnedByScraper ScraperCullAndBinCScraperManifests();
 Superblock ScraperGetSuperblockContract(
     bool bStoreConvergedStats = false,
-    bool bContractDirectFromStatsUpdate = false);
+    bool bContractDirectFromStatsUpdate = false,
+    bool bFromHousekeeping = false);
 
 extern CCriticalSection cs_ConvergedScraperStatsCache;
 extern ConvergedScraperStats ConvergedScraperStatsCache;
@@ -635,11 +636,16 @@ class SuperblockValidator
 
         LogPrintf("ValidateSuperblock(): No match by project.");
 
-        if (OutOfSyncByAge()) {
-            LogPrintf("ValidateSuperblock(): No validation achieved, but node is"
-                      "not in sync - skipping validation.");
+        {
+            LOCK(cs_ConvergedScraperStatsCache);
 
-            return Result::UNKNOWN;
+            if (OutOfSyncByAge() || !ConvergedScraperStatsCache.bMinHousekeepingComplete) {
+                LogPrintf("ValidateSuperblock(): No validation achieved, but node is"
+                          "not in sync or minimum housekeeping is not complete"
+                          " - skipping validation.");
+
+                return Result::UNKNOWN;
+            }
         }
 
         return Result::INVALID;

src/gridcoin/scraper/scraper.cpp

@@ -1277,18 +1277,15 @@ UniValue testnewsb(const UniValue& params, bool fHelp);
 
 bool ScraperHousekeeping()
 {
-    // Periodically generate converged manifests and generate SB core and "contract"
-    // This will probably be reduced to the commented out call as we near final testing,
-    // because ScraperGetSuperblockContract(false) is called from the subscriber interface
-    // with the boolean false, meaning don't store the stats.
-    // Lock both cs_Scraper and cs_StructScraperFileManifest.
+    // Periodically generate converged manifests and generate SB contract and store in cache.
 
     Superblock superblock;
 
     {
+        // Lock both cs_Scraper and cs_StructScraperFileManifest.
         LOCK2(cs_Scraper, cs_StructScraperFileManifest);
 
-        superblock = ScraperGetSuperblockContract(true, false);
+        superblock = ScraperGetSuperblockContract(true, false, true);
     }
 
     {
@@ -1327,8 +1324,6 @@ bool ScraperHousekeeping()
     if (log.archive(false, plogfile_out))
         _log(logattribute::INFO, "ScraperHousekeeping", "Archived scraper.log to " + plogfile_out.filename().string());
 
-    //log.closelogfile();
-
     return true;
 }
 
@@ -5056,7 +5051,7 @@ ScraperPendingBeaconMap GetVerifiedBeaconsForReport(bool from_global)
 *      Subscriber      *
 ************************/
 
-Superblock ScraperGetSuperblockContract(bool bStoreConvergedStats, bool bContractDirectFromStatsUpdate)
+Superblock ScraperGetSuperblockContract(bool bStoreConvergedStats, bool bContractDirectFromStatsUpdate, bool bFromHousekeeping)
 {
     Superblock empty_superblock;
 
@@ -5135,6 +5130,9 @@ Superblock ScraperGetSuperblockContract(bool bStoreConvergedStats, bool bContrac
                     // Mark the cache clean, because it was just updated.
                     ConvergedScraperStatsCache.bClean = true;
 
+                    // If called from housekeeping, mark bMinHousekeepingComplete true
+                    if (bFromHousekeeping) ConvergedScraperStatsCache.bMinHousekeepingComplete = true;
+
                     // Signal UI of SBContract status
                     if (superblock.WellFormed())
                     {

src/gridcoin/scraper/scraper.h

@@ -98,7 +98,7 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByConvergedManifest(const Converge
 bool IsScraperAuthorized();
 bool IsScraperAuthorizedToBroadcastManifests(CBitcoinAddress& AddressOut, CKey& KeyOut);
 bool IsScraperMaximumManifestPublishingRateExceeded(int64_t& nTime, CPubKey& PubKey);
-GRC::Superblock ScraperGetSuperblockContract(bool bStoreConvergedStats = false, bool bContractDirectFromStatsUpdate = false);
+GRC::Superblock ScraperGetSuperblockContract(bool bStoreConvergedStats = false, bool bContractDirectFromStatsUpdate = false, bool bFromHousekeeping = false);
 scraperSBvalidationtype ValidateSuperblock(const GRC::Superblock& NewFormatSuperblock, bool bUseCache = true, unsigned int nReducedCacheBits = 32);
 std::vector<uint160> GetVerifiedBeaconIDs(const ConvergedManifest& StructConvergedManifest);
 std::vector<uint160> GetVerifiedBeaconIDs(const ScraperPendingBeaconMap& VerifiedBeaconMap);

src/gridcoin/superblock.h

@@ -1551,6 +1551,7 @@ struct ConvergedScraperStats
     ConvergedScraperStats() : Convergence(), NewFormatSuperblock()
     {
         bClean = false;
+        bMinHousekeepingComplete = false;
 
         nTime = 0;
         mScraperConvergedStats = {};
@@ -1560,6 +1561,7 @@ struct ConvergedScraperStats
     ConvergedScraperStats(const int64_t nTime_in, const ConvergedManifest& Convergence) : Convergence(Convergence)
     {
         bClean = false;
+        bMinHousekeepingComplete = false;
 
         nTime = nTime_in;
 
@@ -1569,7 +1571,18 @@ struct ConvergedScraperStats
     // Flag to indicate cache is clean or dirty (i.e. state change of underlying statistics has occurred.
     // This flag is marked true in ScraperGetSuperblockContract() and false on receipt or deletion of
     // statistics objects.
-    bool bClean = false;
+    bool bClean;
+
+    // This flag tracks the completion of at least one iteration of the housekeeping loop. The purpose of this flag
+    // is to ensure enough time has gone by after a (re)start of the wallet that a complete set of manifests/parts
+    // have been collected. Trying to form a contract too early may result in a local convergence that may not
+    // match an incoming superblock that comes in very close to the wallet start, and if enough manifests/parts are
+    // missing the backup validation checks will fail, resulting in a forked client due to failure to validate
+    // the superblock. This should help the difficult corner case of a wallet restarted literally a minute or two
+    // before the superblock is received. This has the effect of allowing a grace period of nScraperSleep after the
+    // wallet start where an incoming superblock will allowed with Result::UNKNOWN, rather than rejected with
+    // Result::INVALID.
+    bool bMinHousekeepingComplete;
 
     int64_t nTime;
     ScraperStats mScraperConvergedStats;

src/rpcblockchain.cpp

@@ -36,7 +36,9 @@ bool AskForOutstandingBlocks(uint256 hashStart);
 bool ForceReorganizeToHash(uint256 NewHash);
 extern UniValue MagnitudeReport(const GRC::Cpid cpid);
 extern UniValue SuperblockReport(int lookback = 14, bool displaycontract = false, std::string cpid = "");
-extern GRC::Superblock ScraperGetSuperblockContract(bool bStoreConvergedStats = false, bool bContractDirectFromStatsUpdate = false);
+extern GRC::Superblock ScraperGetSuperblockContract(bool bStoreConvergedStats = false,
+                                                    bool bContractDirectFromStatsUpdate = false,
+                                                    bool bFromHousekeeping = false);
 extern ScraperPendingBeaconMap GetPendingBeaconsForReport();
 extern ScraperPendingBeaconMap GetVerifiedBeaconsForReport(bool from_global = false);
 extern UniValue GetJSONVersionReport(const int64_t lookback, const bool full_version);