add checking of the local node's bootstrap status

src/pay/payment_producer.py

@@ -2,9 +2,9 @@
 import csv
 import os
 import threading
-import time
 
 from time import sleep
+from datetime import datetime, timedelta
 from Constants import RunMode, PaymentStatus
 from log_config import main_logger
 from model.reward_log import RewardLog
@@ -22,6 +22,8 @@
 logger = main_logger
 
 MUTEZ = 1e+6
+BOOSTRAP_SLEEP = 32
+
 
 class PaymentProducer(threading.Thread, PaymentProducerABC):
     def __init__(self, name, initial_payment_cycle, network_config, payments_dir, calculations_dir, run_mode,
@@ -39,9 +41,10 @@ def __init__(self, name, initial_payment_cycle, network_config, payments_dir, ca
 
         self.name = name
 
+        self.node_url = node_url
         self.reward_api = provider_factory.newRewardApi(
-            network_config, self.baking_address, node_url, node_url_public, api_base_url)
-        self.block_api = provider_factory.newBlockApi(network_config, node_url, api_base_url)
+            network_config, self.baking_address, self.node_url, node_url_public, api_base_url)
+        self.block_api = provider_factory.newBlockApi(network_config, self.node_url, api_base_url)
 
         self.fee_calc = service_fee_calc
         self.initial_payment_cycle = initial_payment_cycle
@@ -121,6 +124,15 @@ def run(self):
             time.sleep(5)
 
             try:
+
+                # Check if local node is bootstrapped; sleep if needed; restart loop
+                if not node_is_bootstrapped(self.node_url):
+                    logger.info("Local node, {}, is not in sync with the Tezos network. Will sleep for {} blocks and check again."
+                                .format(self.node_url, BOOTSTRAP_SLEEP)
+                    self.wait_until_next_cycle(BOOTSTRAP_SLEEP)
+                    continue
+
+                # Local node is ready
                 current_level = self.block_api.get_current_level(verbose=self.verbose)
                 crrnt_cycle = self.block_api.level_to_cycle(current_level)
 
@@ -284,6 +296,22 @@ def wait_until_next_cycle(self, nb_blocks_remaining):
                 self.exit()
                 break
 
+
+    def node_is_bootstrapped(self):
+        # Get local node's bootstrap time. If bootstrap time + 2 minutes is 
+        # before local time, node is not bootstrapped.
+        try:
+            boot_time = self.block_api.get_bootstrapped()
+            utc_time = datetime.utcnow()
+            if (boot_time + datetime.timedelta(minutes=2)) < utc_time:
+                logger.info("Current time is '{}', latest block of local node is '{}'."
+                            .format(utc_time, boot_time))
+                return False
+        except ValueError:
+            logger.error("Unable to determine local node's bootstrap status. Continuing...")
+        return True
+
+
     def create_calculations_report(self, payment_logs, report_file_path, total_rewards):
         with open(report_file_path, 'w', newline='') as f:
             writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)

src/rpc/rpc_block_api.py

@@ -1,10 +1,12 @@
 import requests
 
+from datetime import datetime
 from api.block_api import BlockApi
 from log_config import main_logger
 
 logger = main_logger
 
+COMM_BOOTSTRAP = "{}/monitor/bootstrapped"
 COMM_HEAD = "{}/chains/main/blocks/head"
 COMM_REVELATION = "{}/chains/main/blocks/head/context/contracts/{}/manager_key"
 
@@ -28,6 +30,13 @@ def get_revelation(self, pkh, verbose=False):
         bool_revelation = manager_key and manager_key!='null'
         return bool_revelation
 
+    def get_bootstrapped(self):
+        response = requests.get(COMM_BOOTSTRAP.format(self.node_url), timeout=5)
+        boot_resp = response.json()
+        boot_time = datetime.strptime(boot_resp["timestamp"], "%Y-%m-%dT%H:%M:%SZ")
+        logger.debug("Local node bootstrap time is '{}'".format(boot_time))
+        return boot_time
+
 
 def test_get_revelation():