introduce "self-ideal" rewards like Backerei (#415)

* rename "ideal" to "expected" rewards
* introduce new "ideal" rewards like Backerei
* ideal: pay rewards assuming perfect behaviour of the baker doing the payouts, while still accounting for imperfect behaviour of other bakers. This acts as a liveness guarantee and ensures maximum payouts for delegators, but without overpaying for other baker's downtime, like the "expected" payout does.
* payment method implemented for all backends
* tzstats does not compensate for missedBlockFees, tzkt does
* Fixed bug where tzkt did pay out slashing rewards. Those rewards are in general returned due to honest mistakes
* Florence fix + handle rpc unfrozen rewards, pre-florence RPC fail
* Contributor: nicolasochem, Effort=Compensated
* Reviewer: jdsika/dansan566, Effort=10h/7h

docs/configuration.rst

@@ -53,10 +53,11 @@ Available configuration parameters are:
     payment_address: tz1boot1pK9h2BVGXdyvfQSv8kd1LQM6H889
 
 **rewards_type**
-  There are two options for calculating the total rewards earned by a baker at the end of each cycle. If this parameter is missing, 'actual' rewards take affect.
+  There are three options for calculating the total rewards earned by a baker at the end of each cycle. If this parameter is missing, 'actual' rewards take affect.
 
-  'ideal': Rewards are calculated using the number of baking rights granted at priority 0, plus the count of all endorsing slots. If a bake or endorsement is missed, rewards are calculated as if there was no miss. No additional block rewards or transaction fees are included in this method.
-  'actual': Rewards are calculated based on the actual number of bakes, at any priority, and any endorsements. If a bake or endorsement is missed, rewards are not earned and therefor not included. Transaction fees and other block rewards are included in the rewards.
+  'actual': Rewards are calculated based on the actual number of bakes, at any priority, and any endorsements. Transaction fees and other block rewards are included in the rewards. If a bake or endorsement is missed or rewards are lost again due to accusations, rewards are not earned and therefore not included.
+  'estimated': Rewards are calculated using the number of baking rights granted at priority 0, plus the count of all endorsing slots. If a bake or endorsement is missed, rewards are calculated as if there was no miss. No additional block rewards or transaction fees are included in this method. This allows to pay future rewards, before the cycle actually runs.
+  'ideal': Rewards are calculated assuming ideal behaviour of the baker, and actual behaviour of other bakers. If a bake or endorsement is missed, rewards are paid out despite not having been earned. Lost income due to other baker downtime is not included. Transaction fees and other block rewards are included in the rewards. Any lost rewards due to double-baking, double endorsing, or missing nonces are compensated at the expense of the baker. Select this type of rewards to insure against downtime of your baker but still account for real world conditions. This way, you will get optimal ranking in baker evaluation services despite any downtime.
 
   Example::
 
@@ -162,4 +163,4 @@ Available configuration parameters are:
 **plugins**
   Please consult the `plugins docs`_ for more details on the configuring the various plugins.
 
-.. _plugins docs : plugins.html
+.. _plugins docs : plugins.html

src/Constants.py

@@ -75,12 +75,16 @@ def __str__(self):
 class RewardsType(Enum):
     ACTUAL = 'actual'
     IDEAL = 'ideal'
+    ESTIMATED = 'estimated'
 
-    def isIdeal(self):
-        return self == RewardsType.IDEAL
+    def isEstimated(self):
+        return self == RewardsType.ESTIMATED
 
     def isActual(self):
         return self == RewardsType.ACTUAL
 
+    def isIdeal(self):
+        return self == RewardsType.IDEAL
+
     def __str__(self):
         return self.value

src/api/reward_api.py

@@ -7,7 +7,7 @@ def __init__(self):
         self.dexter_contracts_set = []
 
     @abstractmethod
-    def get_rewards_for_cycle_map(self, cycle):
+    def get_rewards_for_cycle_map(self, cycle, rewards_type):
         pass
 
     def set_dexter_contracts_set(self, dexter_contracts_set):

src/configure.py

@@ -33,7 +33,7 @@
     'bakingaddress': 'Specify your baking address public key hash (Processing may take a few seconds)',
     'paymentaddress': 'Specify your payouts public key hash. It can be the same as your baking address, or a different one.',
     'servicefee': 'Specify bakery fee [0:100]',
-    'rewardstype': "Specify if baker pays 'ideal', or 'actual' rewards (Be sure to read the documentation to understand the difference). Type enter for 'actual'",
+    'rewardstype': "Specify if baker pays 'ideal', 'estimated' or 'actual' rewards (Be sure to read the documentation to understand the difference). Type enter for 'actual'",
     'foundersmap': "Specify FOUNDERS in form 'PKH1':share1,'PKH2':share2,... (Mind quotes) Type enter to leave empty",
     'ownersmap': "Specify OWNERS in form 'pk1':share1,'pkh2':share2,... (Mind quotes) Type enter to leave empty",
     'mindelegation': "Specify minimum delegation amount in tez. Type enter for 0",
@@ -115,7 +115,7 @@ def onrewardstype(input):
         rt = RewardsType(input.lower())
         parser.set(REWARDS_TYPE, str(rt))
     except Exception:
-        printe("Invalid option for rewards type. Please enter 'actual' or 'ideal'.")
+        printe("Invalid option for rewards type. Please enter 'actual', 'estimated' or 'ideal'.")
         return
 
     fsm.go()

src/pay/payment_consumer.py

@@ -237,7 +237,15 @@ def create_stats_dict(self, nb_failed, nb_unknown, payment_cycle, payment_logs,
         stats_dict['nb_delegators'] = n_d_type
         stats_dict['pay_xfer_fee'] = 1 if self.delegator_pays_xfer_fee else 0
         stats_dict['pay_ra_fee'] = 1 if self.delegator_pays_ra_fee else 0
-        stats_dict['rewards_type'] = "I" if self.rewards_type.isIdeal() else "A"
+        if self.rewards_type.isIdeal():
+            stats_dict['rewards_type'] = "I"
+        elif self.rewards_type.isExpected():
+            stats_dict['rewards_type'] = "E"
+        elif self.rewards_type.isActual():
+            stats_dict['rewards_type'] = "A"
+        else:
+            stats_dict['rewards_type'] = "A"
+            logger.info("Reward type is set to actual by default - please check your configuration")
         stats_dict['trdver'] = str(VERSION)
 
         if self.args:

src/pay/payment_producer.py

@@ -175,8 +175,8 @@ def run(self):
                         # Paying upcoming cycles (-R in [-6, -11] )
                         if pymnt_cycle >= current_cycle:
                             logger.warn("Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation.")
-                            if not self.rewards_type.isIdeal():
-                                logger.error("For future rewards payout, you must configure the payout type to 'Ideal', see documentation")
+                            if not self.rewards_type.isEstimated():
+                                logger.error("For future rewards payout, you must configure the payout type to 'Estimated', see documentation")
                                 self.exit()
                                 break
 
@@ -192,7 +192,7 @@ def run(self):
                             continue  # Break/Repeat loop
 
                         else:
-                            result = self.try_to_pay(pymnt_cycle, expected_rewards=self.rewards_type.isIdeal())
+                            result = self.try_to_pay(pymnt_cycle, self.rewards_type)
 
                         if result:
                             # single run is done. Do not continue.
@@ -250,7 +250,7 @@ def run(self):
 
         return
 
-    def try_to_pay(self, pymnt_cycle, expected_rewards=False):
+    def try_to_pay(self, pymnt_cycle, rewards_type):
         try:
             logger.info("Payment cycle is " + str(pymnt_cycle))
 
@@ -262,12 +262,14 @@ def try_to_pay(self, pymnt_cycle, expected_rewards=False):
                 return True
 
             # 1- get reward data
-            if expected_rewards:
-                logger.info("Using expected/ideal rewards for payouts calculations")
-            else:
+            if rewards_type.isEstimated():
+                logger.info("Using estimated rewards for payouts calculations")
+            elif rewards_type.isActual():
                 logger.info("Using actual rewards for payouts calculations")
+            elif rewards_type.isIdeal():
+                logger.info("Using ideal rewards for payouts calculations")
 
-            reward_model = self.reward_api.get_rewards_for_cycle_map(pymnt_cycle, expected_rewards)
+            reward_model = self.reward_api.get_rewards_for_cycle_map(pymnt_cycle, rewards_type)
 
             # 2- calculate rewards
             reward_logs, total_amount = self.payment_calc.calculate(reward_model)
@@ -288,7 +290,7 @@ def try_to_pay(self, pymnt_cycle, expected_rewards=False):
                 # 6- create calculations report file. This file contains calculations details
                 report_file_path = get_calculation_report_file(self.calculations_dir, pymnt_cycle)
                 logger.debug("Creating calculation report (%s)", report_file_path)
-                self.create_calculations_report(reward_logs, report_file_path, total_amount, expected_rewards)
+                self.create_calculations_report(reward_logs, report_file_path, total_amount, rewards_type)
 
                 # 7- processing of cycle is done
                 logger.info("Reward creation is done for cycle {}, created {} rewards.".format(pymnt_cycle, len(reward_logs)))
@@ -333,9 +335,14 @@ def node_is_bootstrapped(self):
             logger.error("Unable to determine local node's bootstrap status. Continuing...")
         return True
 
-    def create_calculations_report(self, payment_logs, report_file_path, total_rewards, expected_rewards):
+    def create_calculations_report(self, payment_logs, report_file_path, total_rewards, rewards_type):
 
-        rt = "I" if expected_rewards else "A"
+        if rewards_type.isEstimated():
+            rt = "E"
+        elif rewards_type.isActual():
+            rt = "A"
+        elif rewards_type.isIdeal():
+            rt = "I"
 
         # Open reports file and write; auto-closes file
         with open(report_file_path, 'w', newline='') as f:

src/rpc/rpc_reward_api.py

@@ -14,6 +14,8 @@
 COMM_HEAD = "{}/chains/main/blocks/head"
 COMM_DELEGATES = "{}/chains/main/blocks/{}/context/delegates/{}"
 COMM_BLOCK = "{}/chains/main/blocks/{}"
+COMM_BLOCK_METADATA = "{}/chains/main/blocks/{}/metadata"
+COMM_BLOCK_OPERATIONS = "{}/chains/main/blocks/{}/operations"
 COMM_SNAPSHOT = COMM_BLOCK + "/context/raw/json/cycle/{}/roll_snapshot"
 COMM_DELEGATE_BALANCE = "{}/chains/main/blocks/{}/context/contracts/{}/balance"
 COMM_CONTRACT_STORAGE = "{}/chains/main/blocks/{}/context/contracts/{}/storage"
@@ -39,7 +41,7 @@ def __init__(self, nw, baking_address, node_url):
         self.baking_address = baking_address
         self.node_url = node_url
 
-    def get_rewards_for_cycle_map(self, cycle, expected_rewards=False):
+    def get_rewards_for_cycle_map(self, cycle, rewards_type):
         try:
             current_level, current_cycle = self.__get_current_level()
             logger.debug("Current level {:d}, current cycle {:d}".format(current_level, current_cycle))
@@ -58,12 +60,12 @@ def get_rewards_for_cycle_map(self, cycle, expected_rewards=False):
                          .format(cycle, self.preserved_cycles, self.blocks_per_cycle,
                                  level_of_first_block_in_preserved_cycles, level_of_last_block_in_unfreeze_cycle))
 
-            # Decide on if paying actual rewards earned, or paying expected/ideal rewards
-            if expected_rewards:
+            # Decide on if paying actual rewards earned, or paying estimated/ideal rewards
+            if rewards_type.isEstimated():
 
                 # Determine how many priority 0 baking rights delegate had
-                nb_blocks = self.__get_number_of_baking_rights(cycle, level_of_first_block_in_preserved_cycles)
-                nb_endorsements = self.__get_number_of_endorsement_rights(cycle, level_of_first_block_in_preserved_cycles)
+                nb_blocks = len([r for r in self.__get_baking_rights(cycle, level_of_first_block_in_preserved_cycles) if r["priority"] == 0])
+                nb_endorsements = sum([len(r["slots"]) for r in self.__get_endorsement_rights(cycle, level_of_first_block_in_preserved_cycles)])
 
                 logger.debug("Number of 0 priority blocks: {}, Number of endorsements: {}".format(nb_blocks, nb_endorsements))
                 logger.debug("Block reward: {}, Endorsement Reward: {}".format(self.block_reward, self.endorsement_reward))
@@ -73,7 +75,7 @@ def get_rewards_for_cycle_map(self, cycle, expected_rewards=False):
                 total_block_reward = nb_blocks * self.block_reward
                 total_endorsement_reward = nb_endorsements * self.endorsement_reward
 
-                logger.info("Ideal rewards for cycle {:d}, {:,} block rewards ({:d} blocks), {:,} endorsing rewards ({:d} slots)".format(
+                logger.info("Estimated rewards for cycle {:d}, {:,} block rewards ({:d} blocks), {:,} endorsing rewards ({:d} slots)".format(
                             cycle, total_block_reward, nb_blocks, total_endorsement_reward, nb_endorsements))
 
                 reward_data["total_rewards"] = total_block_reward + total_endorsement_reward
@@ -83,10 +85,28 @@ def get_rewards_for_cycle_map(self, cycle, expected_rewards=False):
 
                 if current_level - level_of_last_block_in_unfreeze_cycle >= 0:
                     unfrozen_fees, unfrozen_rewards = self.__get_unfrozen_rewards(level_of_last_block_in_unfreeze_cycle, cycle)
-                    reward_data["total_rewards"] = unfrozen_fees + unfrozen_rewards
+                    total_actual_rewards = unfrozen_fees + unfrozen_rewards
                 else:
                     frozen_fees, frozen_rewards = self.__get_frozen_rewards(cycle, current_level)
-                    reward_data["total_rewards"] = frozen_fees + frozen_rewards
+                    total_actual_rewards = frozen_fees + frozen_rewards
+                if rewards_type.isActual():
+                    reward_data["total_rewards"] = total_actual_rewards
+                elif rewards_type.isIdeal():
+                    missed_baking_income = 0
+                    for r in self.__get_baking_rights(cycle, level_of_first_block_in_preserved_cycles):
+                        if r["priority"] == 0:
+                            if self.__get_block_author(r["level"]) != self.baking_address:
+                                logger.warning("Found missed baking slot {}, adding {} mutez reward anyway.".format(r, self.block_reward))
+                                missed_baking_income += self.block_reward
+                    missed_endorsing_income = 0
+                    for r in self.__get_endorsement_rights(cycle, level_of_first_block_in_preserved_cycles):
+                        authored_endorsement_slots = self.__get_authored_endorsement_slots_by_level(r["level"] + 1)
+                        if authored_endorsement_slots != r["slots"]:
+                            mutez_to_add = self.endorsement_reward * len(r["slots"])
+                            logger.warning("Found {} missed endorsement(s) at level {}, adding {} mutez reward anyway.".format(len(r["slots"]), r["level"], mutez_to_add))
+                            missed_endorsing_income += mutez_to_add
+                    logger.warning("total rewards %s" % (total_actual_rewards + missed_baking_income + missed_endorsing_income))
+                    reward_data["total_rewards"] = total_actual_rewards + missed_baking_income + missed_endorsing_income
 
             # TODO: support Dexter for RPC
             # _, snapshot_level = self.__get_roll_snapshot_block_level(cycle, current_level)
@@ -96,7 +116,7 @@ def get_rewards_for_cycle_map(self, cycle, expected_rewards=False):
             reward_model = RewardProviderModel(reward_data["delegate_staking_balance"], reward_data["total_rewards"],
                                                reward_data["delegators"])
 
-            logger.debug("delegate_staking_balance = {:d}, total_rewards = {:d}"
+            logger.debug("delegate_staking_balance = {:d}, total_rewards = {:f}"
                          .format(reward_data["delegate_staking_balance"], reward_data["total_rewards"]))
             logger.debug("delegators = {}".format(reward_data["delegators"]))
 
@@ -107,37 +127,60 @@ def get_rewards_for_cycle_map(self, cycle, expected_rewards=False):
             # necessary data to properly compute rewards
             raise e from e
 
-    def __get_number_of_baking_rights(self, cycle, level):
+    def __get_baking_rights(self, cycle, level):
+        """
+        Returns list of baking rights for a given cycle.
+        """
 
         try:
             baking_rights_rpc = COMM_BAKING_RIGHTS.format(self.node_url, level, cycle, self.baking_address)
-            baking_rights = self.do_rpc_request(baking_rights_rpc)
+            return self.do_rpc_request(baking_rights_rpc)
 
-            nb_rights = 0
+        except ApiProviderException as e:
+            raise e from e
 
-            # Count all of the priority 0 rights
-            for r in baking_rights:
-                if r["priority"] == 0:
-                    nb_rights += 1
+    def __get_block_author(self, level):
+        """
+        Returns baker public key hash for a given block level.
+        """
 
-            return nb_rights
+        try:
+            block_metadata_rpc = COMM_BLOCK_METADATA.format(self.node_url, level)
+            return self.do_rpc_request(block_metadata_rpc)["baker"]
 
         except ApiProviderException as e:
             raise e from e
 
-    def __get_number_of_endorsement_rights(self, cycle, level):
+    def __get_endorsement_rights(self, cycle, level):
+        """
+        Returns list of endorsements rights for a cycle.
+        """
 
         try:
             endorsing_rights_rpc = COMM_ENDORSING_RIGHTS.format(self.node_url, level, cycle, self.baking_address)
-            endorsing_rights = self.do_rpc_request(endorsing_rights_rpc)
+            return self.do_rpc_request(endorsing_rights_rpc)
 
-            nb_rights = 0
+        except ApiProviderException as e:
+            raise e from e
 
-            # Count all of the slots in each endorsing right
-            for r in endorsing_rights:
-                nb_rights += len(r["slots"])
+    def __get_authored_endorsement_slots_by_level(self, level):
+        """
+        Returns a list of endorsements authored by the baker for a given block level.
+        """
 
-            return nb_rights
+        try:
+            block_operations_rpc = COMM_BLOCK_OPERATIONS.format(self.node_url, level)
+            block_operations = self.do_rpc_request(block_operations_rpc)[0]
+            endorsements = [b for b in block_operations if b["contents"][0]["kind"] == "endorsement_with_slot"]
+            if len(endorsements) == 0:
+                logger.error("Can not parse endorsements from RPC. Aborting.")
+                logger.info("TRD can not process rewards for protocols older than Florence.")
+                raise Exception("Can not parse endorsements from RPC.")
+
+            for e in endorsements:
+                if e["contents"][0]["metadata"]["delegate"] == self.baking_address:
+                    return e["contents"][0]["metadata"]["slots"]
+            return []
 
         except ApiProviderException as e:
             raise e from e