From 53693ab0942357fb9a88164d16c1dbd7197628c6 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Sun, 1 Aug 2021 10:52:49 -0700 Subject: [PATCH] introduce "self-ideal" rewards like Backerei (#415) * rename "ideal" to "expected" rewards * introduce new "ideal" rewards like Backerei * ideal: pay rewards assuming perfect behaviour of the baker doing the payouts, while still accounting for imperfect behaviour of other bakers. This acts as a liveness guarantee and ensures maximum payouts for delegators, but without overpaying for other baker's downtime, like the "expected" payout does. * payment method implemented for all backends * tzstats does not compensate for missedBlockFees, tzkt does * Fixed bug where tzkt did pay out slashing rewards. Those rewards are in general returned due to honest mistakes * Florence fix + handle rpc unfrozen rewards, pre-florence RPC fail * Contributor: nicolasochem, Effort=Compensated * Reviewer: jdsika/dansan566, Effort=10h/7h --- docs/configuration.rst | 9 +- src/Constants.py | 8 +- src/api/reward_api.py | 2 +- src/configure.py | 4 +- src/pay/payment_consumer.py | 10 +- src/pay/payment_producer.py | 29 +++--- src/rpc/rpc_reward_api.py | 91 ++++++++++++++----- src/tzkt/tzkt_reward_api.py | 53 +++++++---- src/tzstats/tzstats_reward_api.py | 4 +- src/tzstats/tzstats_reward_provider_helper.py | 32 ++++--- tests/integration/test_phases.py | 3 +- tests/integration/test_tzkt_reward_api.py | 13 +-- .../test_gas_estimation_oven_kt1.py | 4 +- tests/unit/test_calculatePhase0.py | 2 +- 14 files changed, 180 insertions(+), 84 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index c7fcc042..0f978474 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -53,10 +53,11 @@ Available configuration parameters are: payment_address: tz1boot1pK9h2BVGXdyvfQSv8kd1LQM6H889 **rewards_type** - There are two options for calculating the total rewards earned by a baker at the end of each cycle. If this parameter is missing, 'actual' rewards take affect. + There are three options for calculating the total rewards earned by a baker at the end of each cycle. If this parameter is missing, 'actual' rewards take affect. - 'ideal': Rewards are calculated using the number of baking rights granted at priority 0, plus the count of all endorsing slots. If a bake or endorsement is missed, rewards are calculated as if there was no miss. No additional block rewards or transaction fees are included in this method. - 'actual': Rewards are calculated based on the actual number of bakes, at any priority, and any endorsements. If a bake or endorsement is missed, rewards are not earned and therefor not included. Transaction fees and other block rewards are included in the rewards. + 'actual': Rewards are calculated based on the actual number of bakes, at any priority, and any endorsements. Transaction fees and other block rewards are included in the rewards. If a bake or endorsement is missed or rewards are lost again due to accusations, rewards are not earned and therefore not included. + 'estimated': Rewards are calculated using the number of baking rights granted at priority 0, plus the count of all endorsing slots. If a bake or endorsement is missed, rewards are calculated as if there was no miss. No additional block rewards or transaction fees are included in this method. This allows to pay future rewards, before the cycle actually runs. + 'ideal': Rewards are calculated assuming ideal behaviour of the baker, and actual behaviour of other bakers. If a bake or endorsement is missed, rewards are paid out despite not having been earned. Lost income due to other baker downtime is not included. Transaction fees and other block rewards are included in the rewards. Any lost rewards due to double-baking, double endorsing, or missing nonces are compensated at the expense of the baker. Select this type of rewards to insure against downtime of your baker but still account for real world conditions. This way, you will get optimal ranking in baker evaluation services despite any downtime. Example:: @@ -162,4 +163,4 @@ Available configuration parameters are: **plugins** Please consult the `plugins docs`_ for more details on the configuring the various plugins. -.. _plugins docs : plugins.html \ No newline at end of file +.. _plugins docs : plugins.html diff --git a/src/Constants.py b/src/Constants.py index 57a1365a..51537c5e 100644 --- a/src/Constants.py +++ b/src/Constants.py @@ -75,12 +75,16 @@ def __str__(self): class RewardsType(Enum): ACTUAL = 'actual' IDEAL = 'ideal' + ESTIMATED = 'estimated' - def isIdeal(self): - return self == RewardsType.IDEAL + def isEstimated(self): + return self == RewardsType.ESTIMATED def isActual(self): return self == RewardsType.ACTUAL + def isIdeal(self): + return self == RewardsType.IDEAL + def __str__(self): return self.value diff --git a/src/api/reward_api.py b/src/api/reward_api.py index ddc6f1b6..e8a35d08 100644 --- a/src/api/reward_api.py +++ b/src/api/reward_api.py @@ -7,7 +7,7 @@ def __init__(self): self.dexter_contracts_set = [] @abstractmethod - def get_rewards_for_cycle_map(self, cycle): + def get_rewards_for_cycle_map(self, cycle, rewards_type): pass def set_dexter_contracts_set(self, dexter_contracts_set): diff --git a/src/configure.py b/src/configure.py index dd923152..b358da0b 100644 --- a/src/configure.py +++ b/src/configure.py @@ -33,7 +33,7 @@ 'bakingaddress': 'Specify your baking address public key hash (Processing may take a few seconds)', 'paymentaddress': 'Specify your payouts public key hash. It can be the same as your baking address, or a different one.', 'servicefee': 'Specify bakery fee [0:100]', - 'rewardstype': "Specify if baker pays 'ideal', or 'actual' rewards (Be sure to read the documentation to understand the difference). Type enter for 'actual'", + 'rewardstype': "Specify if baker pays 'ideal', 'estimated' or 'actual' rewards (Be sure to read the documentation to understand the difference). Type enter for 'actual'", 'foundersmap': "Specify FOUNDERS in form 'PKH1':share1,'PKH2':share2,... (Mind quotes) Type enter to leave empty", 'ownersmap': "Specify OWNERS in form 'pk1':share1,'pkh2':share2,... (Mind quotes) Type enter to leave empty", 'mindelegation': "Specify minimum delegation amount in tez. Type enter for 0", @@ -115,7 +115,7 @@ def onrewardstype(input): rt = RewardsType(input.lower()) parser.set(REWARDS_TYPE, str(rt)) except Exception: - printe("Invalid option for rewards type. Please enter 'actual' or 'ideal'.") + printe("Invalid option for rewards type. Please enter 'actual', 'estimated' or 'ideal'.") return fsm.go() diff --git a/src/pay/payment_consumer.py b/src/pay/payment_consumer.py index 5f7e23fc..add14ba6 100644 --- a/src/pay/payment_consumer.py +++ b/src/pay/payment_consumer.py @@ -237,7 +237,15 @@ def create_stats_dict(self, nb_failed, nb_unknown, payment_cycle, payment_logs, stats_dict['nb_delegators'] = n_d_type stats_dict['pay_xfer_fee'] = 1 if self.delegator_pays_xfer_fee else 0 stats_dict['pay_ra_fee'] = 1 if self.delegator_pays_ra_fee else 0 - stats_dict['rewards_type'] = "I" if self.rewards_type.isIdeal() else "A" + if self.rewards_type.isIdeal(): + stats_dict['rewards_type'] = "I" + elif self.rewards_type.isExpected(): + stats_dict['rewards_type'] = "E" + elif self.rewards_type.isActual(): + stats_dict['rewards_type'] = "A" + else: + stats_dict['rewards_type'] = "A" + logger.info("Reward type is set to actual by default - please check your configuration") stats_dict['trdver'] = str(VERSION) if self.args: diff --git a/src/pay/payment_producer.py b/src/pay/payment_producer.py index ac8a7980..cfe4db31 100644 --- a/src/pay/payment_producer.py +++ b/src/pay/payment_producer.py @@ -175,8 +175,8 @@ def run(self): # Paying upcoming cycles (-R in [-6, -11] ) if pymnt_cycle >= current_cycle: logger.warn("Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation.") - if not self.rewards_type.isIdeal(): - logger.error("For future rewards payout, you must configure the payout type to 'Ideal', see documentation") + if not self.rewards_type.isEstimated(): + logger.error("For future rewards payout, you must configure the payout type to 'Estimated', see documentation") self.exit() break @@ -192,7 +192,7 @@ def run(self): continue # Break/Repeat loop else: - result = self.try_to_pay(pymnt_cycle, expected_rewards=self.rewards_type.isIdeal()) + result = self.try_to_pay(pymnt_cycle, self.rewards_type) if result: # single run is done. Do not continue. @@ -250,7 +250,7 @@ def run(self): return - def try_to_pay(self, pymnt_cycle, expected_rewards=False): + def try_to_pay(self, pymnt_cycle, rewards_type): try: logger.info("Payment cycle is " + str(pymnt_cycle)) @@ -262,12 +262,14 @@ def try_to_pay(self, pymnt_cycle, expected_rewards=False): return True # 1- get reward data - if expected_rewards: - logger.info("Using expected/ideal rewards for payouts calculations") - else: + if rewards_type.isEstimated(): + logger.info("Using estimated rewards for payouts calculations") + elif rewards_type.isActual(): logger.info("Using actual rewards for payouts calculations") + elif rewards_type.isIdeal(): + logger.info("Using ideal rewards for payouts calculations") - reward_model = self.reward_api.get_rewards_for_cycle_map(pymnt_cycle, expected_rewards) + reward_model = self.reward_api.get_rewards_for_cycle_map(pymnt_cycle, rewards_type) # 2- calculate rewards reward_logs, total_amount = self.payment_calc.calculate(reward_model) @@ -288,7 +290,7 @@ def try_to_pay(self, pymnt_cycle, expected_rewards=False): # 6- create calculations report file. This file contains calculations details report_file_path = get_calculation_report_file(self.calculations_dir, pymnt_cycle) logger.debug("Creating calculation report (%s)", report_file_path) - self.create_calculations_report(reward_logs, report_file_path, total_amount, expected_rewards) + self.create_calculations_report(reward_logs, report_file_path, total_amount, rewards_type) # 7- processing of cycle is done logger.info("Reward creation is done for cycle {}, created {} rewards.".format(pymnt_cycle, len(reward_logs))) @@ -333,9 +335,14 @@ def node_is_bootstrapped(self): logger.error("Unable to determine local node's bootstrap status. Continuing...") return True - def create_calculations_report(self, payment_logs, report_file_path, total_rewards, expected_rewards): + def create_calculations_report(self, payment_logs, report_file_path, total_rewards, rewards_type): - rt = "I" if expected_rewards else "A" + if rewards_type.isEstimated(): + rt = "E" + elif rewards_type.isActual(): + rt = "A" + elif rewards_type.isIdeal(): + rt = "I" # Open reports file and write; auto-closes file with open(report_file_path, 'w', newline='') as f: diff --git a/src/rpc/rpc_reward_api.py b/src/rpc/rpc_reward_api.py index 58c895a0..65047370 100644 --- a/src/rpc/rpc_reward_api.py +++ b/src/rpc/rpc_reward_api.py @@ -14,6 +14,8 @@ COMM_HEAD = "{}/chains/main/blocks/head" COMM_DELEGATES = "{}/chains/main/blocks/{}/context/delegates/{}" COMM_BLOCK = "{}/chains/main/blocks/{}" +COMM_BLOCK_METADATA = "{}/chains/main/blocks/{}/metadata" +COMM_BLOCK_OPERATIONS = "{}/chains/main/blocks/{}/operations" COMM_SNAPSHOT = COMM_BLOCK + "/context/raw/json/cycle/{}/roll_snapshot" COMM_DELEGATE_BALANCE = "{}/chains/main/blocks/{}/context/contracts/{}/balance" COMM_CONTRACT_STORAGE = "{}/chains/main/blocks/{}/context/contracts/{}/storage" @@ -39,7 +41,7 @@ def __init__(self, nw, baking_address, node_url): self.baking_address = baking_address self.node_url = node_url - def get_rewards_for_cycle_map(self, cycle, expected_rewards=False): + def get_rewards_for_cycle_map(self, cycle, rewards_type): try: current_level, current_cycle = self.__get_current_level() logger.debug("Current level {:d}, current cycle {:d}".format(current_level, current_cycle)) @@ -58,12 +60,12 @@ def get_rewards_for_cycle_map(self, cycle, expected_rewards=False): .format(cycle, self.preserved_cycles, self.blocks_per_cycle, level_of_first_block_in_preserved_cycles, level_of_last_block_in_unfreeze_cycle)) - # Decide on if paying actual rewards earned, or paying expected/ideal rewards - if expected_rewards: + # Decide on if paying actual rewards earned, or paying estimated/ideal rewards + if rewards_type.isEstimated(): # Determine how many priority 0 baking rights delegate had - nb_blocks = self.__get_number_of_baking_rights(cycle, level_of_first_block_in_preserved_cycles) - nb_endorsements = self.__get_number_of_endorsement_rights(cycle, level_of_first_block_in_preserved_cycles) + nb_blocks = len([r for r in self.__get_baking_rights(cycle, level_of_first_block_in_preserved_cycles) if r["priority"] == 0]) + nb_endorsements = sum([len(r["slots"]) for r in self.__get_endorsement_rights(cycle, level_of_first_block_in_preserved_cycles)]) logger.debug("Number of 0 priority blocks: {}, Number of endorsements: {}".format(nb_blocks, nb_endorsements)) logger.debug("Block reward: {}, Endorsement Reward: {}".format(self.block_reward, self.endorsement_reward)) @@ -73,7 +75,7 @@ def get_rewards_for_cycle_map(self, cycle, expected_rewards=False): total_block_reward = nb_blocks * self.block_reward total_endorsement_reward = nb_endorsements * self.endorsement_reward - logger.info("Ideal rewards for cycle {:d}, {:,} block rewards ({:d} blocks), {:,} endorsing rewards ({:d} slots)".format( + logger.info("Estimated rewards for cycle {:d}, {:,} block rewards ({:d} blocks), {:,} endorsing rewards ({:d} slots)".format( cycle, total_block_reward, nb_blocks, total_endorsement_reward, nb_endorsements)) reward_data["total_rewards"] = total_block_reward + total_endorsement_reward @@ -83,10 +85,28 @@ def get_rewards_for_cycle_map(self, cycle, expected_rewards=False): if current_level - level_of_last_block_in_unfreeze_cycle >= 0: unfrozen_fees, unfrozen_rewards = self.__get_unfrozen_rewards(level_of_last_block_in_unfreeze_cycle, cycle) - reward_data["total_rewards"] = unfrozen_fees + unfrozen_rewards + total_actual_rewards = unfrozen_fees + unfrozen_rewards else: frozen_fees, frozen_rewards = self.__get_frozen_rewards(cycle, current_level) - reward_data["total_rewards"] = frozen_fees + frozen_rewards + total_actual_rewards = frozen_fees + frozen_rewards + if rewards_type.isActual(): + reward_data["total_rewards"] = total_actual_rewards + elif rewards_type.isIdeal(): + missed_baking_income = 0 + for r in self.__get_baking_rights(cycle, level_of_first_block_in_preserved_cycles): + if r["priority"] == 0: + if self.__get_block_author(r["level"]) != self.baking_address: + logger.warning("Found missed baking slot {}, adding {} mutez reward anyway.".format(r, self.block_reward)) + missed_baking_income += self.block_reward + missed_endorsing_income = 0 + for r in self.__get_endorsement_rights(cycle, level_of_first_block_in_preserved_cycles): + authored_endorsement_slots = self.__get_authored_endorsement_slots_by_level(r["level"] + 1) + if authored_endorsement_slots != r["slots"]: + mutez_to_add = self.endorsement_reward * len(r["slots"]) + logger.warning("Found {} missed endorsement(s) at level {}, adding {} mutez reward anyway.".format(len(r["slots"]), r["level"], mutez_to_add)) + missed_endorsing_income += mutez_to_add + logger.warning("total rewards %s" % (total_actual_rewards + missed_baking_income + missed_endorsing_income)) + reward_data["total_rewards"] = total_actual_rewards + missed_baking_income + missed_endorsing_income # TODO: support Dexter for RPC # _, snapshot_level = self.__get_roll_snapshot_block_level(cycle, current_level) @@ -96,7 +116,7 @@ def get_rewards_for_cycle_map(self, cycle, expected_rewards=False): reward_model = RewardProviderModel(reward_data["delegate_staking_balance"], reward_data["total_rewards"], reward_data["delegators"]) - logger.debug("delegate_staking_balance = {:d}, total_rewards = {:d}" + logger.debug("delegate_staking_balance = {:d}, total_rewards = {:f}" .format(reward_data["delegate_staking_balance"], reward_data["total_rewards"])) logger.debug("delegators = {}".format(reward_data["delegators"])) @@ -107,37 +127,60 @@ def get_rewards_for_cycle_map(self, cycle, expected_rewards=False): # necessary data to properly compute rewards raise e from e - def __get_number_of_baking_rights(self, cycle, level): + def __get_baking_rights(self, cycle, level): + """ + Returns list of baking rights for a given cycle. + """ try: baking_rights_rpc = COMM_BAKING_RIGHTS.format(self.node_url, level, cycle, self.baking_address) - baking_rights = self.do_rpc_request(baking_rights_rpc) + return self.do_rpc_request(baking_rights_rpc) - nb_rights = 0 + except ApiProviderException as e: + raise e from e - # Count all of the priority 0 rights - for r in baking_rights: - if r["priority"] == 0: - nb_rights += 1 + def __get_block_author(self, level): + """ + Returns baker public key hash for a given block level. + """ - return nb_rights + try: + block_metadata_rpc = COMM_BLOCK_METADATA.format(self.node_url, level) + return self.do_rpc_request(block_metadata_rpc)["baker"] except ApiProviderException as e: raise e from e - def __get_number_of_endorsement_rights(self, cycle, level): + def __get_endorsement_rights(self, cycle, level): + """ + Returns list of endorsements rights for a cycle. + """ try: endorsing_rights_rpc = COMM_ENDORSING_RIGHTS.format(self.node_url, level, cycle, self.baking_address) - endorsing_rights = self.do_rpc_request(endorsing_rights_rpc) + return self.do_rpc_request(endorsing_rights_rpc) - nb_rights = 0 + except ApiProviderException as e: + raise e from e - # Count all of the slots in each endorsing right - for r in endorsing_rights: - nb_rights += len(r["slots"]) + def __get_authored_endorsement_slots_by_level(self, level): + """ + Returns a list of endorsements authored by the baker for a given block level. + """ - return nb_rights + try: + block_operations_rpc = COMM_BLOCK_OPERATIONS.format(self.node_url, level) + block_operations = self.do_rpc_request(block_operations_rpc)[0] + endorsements = [b for b in block_operations if b["contents"][0]["kind"] == "endorsement_with_slot"] + if len(endorsements) == 0: + logger.error("Can not parse endorsements from RPC. Aborting.") + logger.info("TRD can not process rewards for protocols older than Florence.") + raise Exception("Can not parse endorsements from RPC.") + + for e in endorsements: + if e["contents"][0]["metadata"]["delegate"] == self.baking_address: + return e["contents"][0]["metadata"]["slots"] + return [] except ApiProviderException as e: raise e from e diff --git a/src/tzkt/tzkt_reward_api.py b/src/tzkt/tzkt_reward_api.py index 8c359c4c..eee42bab 100644 --- a/src/tzkt/tzkt_reward_api.py +++ b/src/tzkt/tzkt_reward_api.py @@ -18,9 +18,9 @@ def __init__(self, nw, baking_address, base_url=None): self.baking_address = baking_address self.name = 'tzkt' - def calc_expected_reward(self, cycle: int, num_blocks: int, num_endorsements: int) -> int: + def calc_estimated_reward(self, cycle: int, num_blocks: int, num_endorsements: int) -> int: """ - Calculate ideal rewards (0 priority, 32 endorsements per block) based on baking rights only + Calculate estimated rewards (0 priority, 32 endorsements per block) based on baking rights only :param cycle: Cycle :param num_blocks: Number of baking rights :param num_endorsements: Number of endorsement rights @@ -36,11 +36,11 @@ def calc_expected_reward(self, cycle: int, num_blocks: int, num_endorsements: in return num_blocks * block_reward + num_endorsements * endorsement_reward - def get_rewards_for_cycle_map(self, cycle, expected_reward=False) -> RewardProviderModel: + def get_rewards_for_cycle_map(self, cycle, rewards_type) -> RewardProviderModel: """ Returns reward split in a specified format :param cycle: - :param expected_reward: + :param rewards_type: :return: RewardProviderModel( delegate_staking_balance=5265698993303, total_reward_amount=2790471275, @@ -56,7 +56,7 @@ def get_rewards_for_cycle_map(self, cycle, expected_reward=False) -> RewardProvi delegate_staking_balance = split['stakingBalance'] - if expected_reward: + if rewards_type.isEstimated(): num_blocks = \ split['ownBlocks'] \ + split['missedOwnBlocks'] \ @@ -69,23 +69,44 @@ def get_rewards_for_cycle_map(self, cycle, expected_reward=False) -> RewardProvi + split['uncoveredEndorsements'] \ + split['futureEndorsements'] - total_reward_amount = self.calc_expected_reward(cycle, num_blocks, num_endorsements) + total_reward_amount = self.calc_estimated_reward(cycle, num_blocks, num_endorsements) else: - total_reward_amount = \ + # rewards earned (excluding equivocation losses) + total_rewards_and_fees = \ split['ownBlockRewards'] \ + split['extraBlockRewards'] \ + split['endorsementRewards'] \ + split['ownBlockFees'] \ + split['extraBlockFees'] \ - + split['revelationRewards'] \ - - split['doubleBakingLostDeposits'] \ - - split['doubleBakingLostRewards'] \ - - split['doubleBakingLostFees'] \ - - split['doubleEndorsingLostDeposits'] \ - - split['doubleEndorsingLostRewards'] \ - - split['doubleEndorsingLostFees'] \ - - split['revelationLostRewards'] \ - - split['revelationLostFees'] + + split['revelationRewards'] + # slashing denunciation rewards are not part of any calculation for now: + # + split['doubleBakingRewards'] \ + # + split['doubleEndorsingRewards'] + # Rationale: normally bakers return those funds to the one slashed in case of an honest mistake + # TODO: make it configurable + total_equivocation_losses = split['doubleBakingLostDeposits'] \ + + split['doubleBakingLostRewards'] \ + + split['doubleBakingLostFees'] \ + + split['doubleEndorsingLostDeposits'] \ + + split['doubleEndorsingLostRewards'] \ + + split['doubleEndorsingLostFees'] \ + + split['revelationLostRewards'] \ + + split['revelationLostFees'] + # losses due to being offline or not having enough bond + total_offline_losses = split['missedOwnBlockRewards'] \ + + split['missedExtraBlockRewards'] \ + + split['uncoveredOwnBlockRewards'] \ + + split['uncoveredExtraBlockRewards'] \ + + split['missedEndorsementRewards'] \ + + split['uncoveredEndorsementRewards'] \ + + split['missedOwnBlockFees'] \ + + split['missedExtraBlockFees'] \ + + split['uncoveredOwnBlockFees'] \ + + split['uncoveredExtraBlockFees'] + if rewards_type.isActual(): + total_reward_amount = total_rewards_and_fees - total_equivocation_losses + elif rewards_type.isIdeal(): + total_reward_amount = total_rewards_and_fees + total_offline_losses total_reward_amount = max(0, total_reward_amount) diff --git a/src/tzstats/tzstats_reward_api.py b/src/tzstats/tzstats_reward_api.py index 6cf4a887..f76753cb 100644 --- a/src/tzstats/tzstats_reward_api.py +++ b/src/tzstats/tzstats_reward_api.py @@ -16,9 +16,9 @@ def __init__(self, nw, baking_address): self.logger = main_logger self.helper = TzStatsRewardProviderHelper(nw, baking_address) - def get_rewards_for_cycle_map(self, cycle, expected_reward=False): + def get_rewards_for_cycle_map(self, cycle, rewards_type): - root = self.helper.get_rewards_for_cycle(cycle, expected_reward) + root = self.helper.get_rewards_for_cycle(cycle, rewards_type) delegate_staking_balance = root["delegate_staking_balance"] total_reward_amount = root["total_reward_amount"] diff --git a/src/tzstats/tzstats_reward_provider_helper.py b/src/tzstats/tzstats_reward_provider_helper.py index 0bb31800..c65aa813 100644 --- a/src/tzstats/tzstats_reward_provider_helper.py +++ b/src/tzstats/tzstats_reward_provider_helper.py @@ -8,7 +8,8 @@ from tzstats.tzstats_api_constants import idx_income_expected_income, idx_income_baking_income, idx_income_endorsing_income, \ idx_income_seed_income, idx_income_fees_income, idx_income_lost_accusation_fees, idx_income_lost_accusation_rewards, \ idx_income_lost_revelation_fees, idx_income_lost_revelation_rewards, idx_delegator_address, idx_balance, \ - idx_baker_delegated, idx_cb_delegator_address, idx_cb_current_balance + idx_baker_delegated, idx_cb_delegator_address, idx_cb_current_balance, idx_income_missed_baking_income, \ + idx_income_missed_endorsing_income from Constants import TZSTATS_PREFIX_API logger = main_logger @@ -42,7 +43,7 @@ def __init__(self, nw, baking_address): self.baking_address = baking_address - def get_rewards_for_cycle(self, cycle, expected_rewards=False): + def get_rewards_for_cycle(self, cycle, rewards_type): root = {"delegate_staking_balance": 0, "total_reward_amount": 0, "delegators_balances": {}} @@ -64,17 +65,26 @@ def get_rewards_for_cycle(self, cycle, expected_rewards=False): raise ApiProviderException('GET {} {}'.format(uri, resp.status_code)) resp = resp.json()[0] - if expected_rewards: + if rewards_type.isEstimated(): root["total_reward_amount"] = int(1e6 * float(resp[idx_income_expected_income])) else: - root["total_reward_amount"] = int(1e6 * (float(resp[idx_income_baking_income]) - + float(resp[idx_income_endorsing_income]) - + float(resp[idx_income_seed_income]) - + float(resp[idx_income_fees_income]) - - float(resp[idx_income_lost_accusation_fees]) - - float(resp[idx_income_lost_accusation_rewards]) - - float(resp[idx_income_lost_revelation_fees]) - - float(resp[idx_income_lost_revelation_rewards]))) + # rewards earned (excluding equivocation losses and equivocation accusation income) + total_rewards_and_fees = (float(resp[idx_income_baking_income]) + + float(resp[idx_income_endorsing_income]) + + float(resp[idx_income_seed_income]) + + float(resp[idx_income_fees_income])) + # losses due to baker double baking, double endorsing or missing nonce + total_equivocation_losses = (float(resp[idx_income_lost_accusation_fees]) + + float(resp[idx_income_lost_accusation_rewards]) + + float(resp[idx_income_lost_revelation_fees]) + + float(resp[idx_income_lost_revelation_rewards])) + # losses due to being offline or not having enough bond + total_offline_losses = (float(resp[idx_income_missed_baking_income]) + + float(resp[idx_income_missed_endorsing_income])) + if rewards_type.isActual(): + root["total_reward_amount"] = int(1e6 * (total_rewards_and_fees - total_equivocation_losses)) + elif rewards_type.isIdeal(): + root["total_reward_amount"] = int(1e6 * (total_rewards_and_fees + total_offline_losses)) # # Get staking balances of delegators at snapshot block diff --git a/tests/integration/test_phases.py b/tests/integration/test_phases.py index d92f293b..90df0eb0 100644 --- a/tests/integration/test_phases.py +++ b/tests/integration/test_phases.py @@ -4,6 +4,7 @@ from unittest.mock import patch, MagicMock from Constants import CURRENT_TESTNET +from Constants import RewardsType from api.provider_factory import ProviderFactory from calc.phased_payment_calculator import PhasedPaymentCalculator from calc.calculate_phaseMapping import CalculatePhaseMapping @@ -84,7 +85,7 @@ def test_process_payouts(self): try: # Reward data # Fetch cycle 90 of delphinet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V - reward_model = rewardApi.get_rewards_for_cycle_map(PAYOUT_CYCLE) + reward_model = rewardApi.get_rewards_for_cycle_map(PAYOUT_CYCLE, RewardsType.ACTUAL) # Calculate rewards - payment_producer.py reward_logs, total_amount = payment_calc.calculate(reward_model) diff --git a/tests/integration/test_tzkt_reward_api.py b/tests/integration/test_tzkt_reward_api.py index cdb52bec..95ed1f08 100644 --- a/tests/integration/test_tzkt_reward_api.py +++ b/tests/integration/test_tzkt_reward_api.py @@ -5,6 +5,7 @@ from unittest.mock import patch, MagicMock from os.path import dirname, join +from Constants import RewardsType from rpc.rpc_reward_api import RpcRewardApiImpl from tzstats.tzstats_reward_api import TzStatsRewardApiImpl, RewardProviderModel from tzkt.tzkt_reward_api import TzKTRewardApiImpl, RewardLog @@ -75,13 +76,13 @@ def test_get_rewards_for_cycle_map(self, address, cycle): nw=default_network_config_map['MAINNET'], baking_address=address, node_url='https://rpc.tzkt.io/mainnet') - rpc_rewards = rpc_impl.get_rewards_for_cycle_map(cycle) + rpc_rewards = rpc_impl.get_rewards_for_cycle_map(cycle, RewardsType.ACTUAL) store_reward_model(address, cycle, 'actual', rpc_rewards) tzkt_impl = TzKTRewardApiImpl( nw=default_network_config_map['MAINNET'], baking_address=address) - tzkt_rewards = tzkt_impl.get_rewards_for_cycle_map(cycle) + tzkt_rewards = tzkt_impl.get_rewards_for_cycle_map(cycle, RewardsType.ACTUAL) self.assertAlmostEqual(rpc_rewards.delegate_staking_balance, tzkt_rewards.delegate_staking_balance, delta=1) self.assertAlmostEqual(rpc_rewards.total_reward_amount, tzkt_rewards.total_reward_amount, delta=1) @@ -100,13 +101,13 @@ def test_expected_rewards(self, address, cycle): tzstats_impl = TzStatsRewardApiImpl( nw=default_network_config_map['MAINNET'], baking_address=address) - tzstats_rewards = tzstats_impl.get_rewards_for_cycle_map(cycle, expected_reward=True) + tzstats_rewards = tzstats_impl.get_rewards_for_cycle_map(cycle, RewardsType.ESTIMATED) store_reward_model(address, cycle, 'expected', tzstats_rewards) tzkt_impl = TzKTRewardApiImpl( nw=default_network_config_map['MAINNET'], baking_address=address) - tzkt_rewards = tzkt_impl.get_rewards_for_cycle_map(cycle, expected_reward=True) + tzkt_rewards = tzkt_impl.get_rewards_for_cycle_map(cycle, RewardsType.ESTIMATED) self.assertAlmostEqual( tzstats_rewards.delegate_staking_balance, tzkt_rewards.delegate_staking_balance, delta=1) @@ -125,13 +126,13 @@ def test_staking_balance_issue(self): nw=default_network_config_map['MAINNET'], baking_address=address, node_url='https://rpc.tzkt.io/mainnet') - rpc_rewards = rpc_impl.get_rewards_for_cycle_map(cycle) + rpc_rewards = rpc_impl.get_rewards_for_cycle_map(cycle, RewardsType.ACTUAL) store_reward_model(address, cycle, 'actual', rpc_rewards) tzkt_impl = TzKTRewardApiImpl( nw=default_network_config_map['MAINNET'], baking_address=address) - tzkt_rewards = tzkt_impl.get_rewards_for_cycle_map(cycle) + tzkt_rewards = tzkt_impl.get_rewards_for_cycle_map(cycle, RewardsType.ACTUAL) self.assertNotEqual(rpc_rewards.delegate_staking_balance, tzkt_rewards.delegate_staking_balance) self.assertAlmostEqual(rpc_rewards.total_reward_amount, tzkt_rewards.total_reward_amount, delta=1) diff --git a/tests/regression/test_gas_estimation_oven_kt1.py b/tests/regression/test_gas_estimation_oven_kt1.py index 05edd748..cba0aea2 100644 --- a/tests/regression/test_gas_estimation_oven_kt1.py +++ b/tests/regression/test_gas_estimation_oven_kt1.py @@ -4,7 +4,7 @@ from pay.batch_payer import BatchPayer from cli.client_manager import ClientManager -from Constants import CURRENT_TESTNET, PUBLIC_NODE_URL +from Constants import CURRENT_TESTNET, PUBLIC_NODE_URL, RewardsType from api.provider_factory import ProviderFactory from config.yaml_baking_conf_parser import BakingYamlConfParser from model.baking_conf import BakingConf @@ -100,7 +100,7 @@ def test_batch_payer_total_payout_amount(): # Reward data # Fetch cycle 90 of delphinet for tz1gtHbmBF3TSebsgJfJPvUB2e9x8EDeNm6V - reward_model = rewardApi.get_rewards_for_cycle_map(PAYOUT_CYCLE) + reward_model = rewardApi.get_rewards_for_cycle_map(PAYOUT_CYCLE, RewardsType.ACTUAL) # Calculate rewards - payment_producer.py reward_logs, total_amount = payment_calc.calculate(reward_model) diff --git a/tests/unit/test_calculatePhase0.py b/tests/unit/test_calculatePhase0.py index 4e413373..d0269577 100644 --- a/tests/unit/test_calculatePhase0.py +++ b/tests/unit/test_calculatePhase0.py @@ -26,7 +26,7 @@ def test_calculate(self): api = ProviderFactory(provider='prpc').newRewardApi(nw, BAKING_ADDRESS, '') - model = api.get_rewards_for_cycle_map(11) + model = api.get_rewards_for_cycle_map(11, 'actual') phase0 = CalculatePhase0(model) reward_data, total_rewards = phase0.calculate()