From 19a742494e78f2dc3863de803596cc7d8de27ee3 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Thu, 12 Nov 2020 10:49:12 -0800 Subject: [PATCH 01/18] Add chain var proposal for hip17 - Calculate hex densities - Add test for calculated hex densities on ledger pinned at 587624 - Add scale_test and specs - Add scale function for density - Add scale_test - Remove unnecessary hip17 chain vars - Plug HIP17 calculations to reward txn - Introduce yet another chain var for density_target_res - Add scale for rx and tx rewards --- include/blockchain_vars.hrl | 28 ++ src/blockchain_hex.erl | 217 ++++++++++++++++ .../v1/blockchain_txn_rewards_v1.erl | 144 ++++++++--- .../v1/blockchain_txn_vars_v1.erl | 90 +++++++ test/blockchain_ct_utils.erl | 74 +++++- test/blockchain_hex_SUITE.erl | 241 ++++++++++++++++++ 6 files changed, 762 insertions(+), 32 deletions(-) create mode 100644 src/blockchain_hex.erl create mode 100644 test/blockchain_hex_SUITE.erl diff --git a/include/blockchain_vars.hrl b/include/blockchain_vars.hrl index b1eef14369..2e720bedc4 100644 --- a/include/blockchain_vars.hrl +++ b/include/blockchain_vars.hrl @@ -370,3 +370,31 @@ %% POC challenge for X blocks would be considered stale for the purposes %% of a hotspot transfer. (We do not allow stale hotspots to be transferred.) -define(transfer_hotspot_stale_poc_blocks, transfer_hotspot_stale_poc_blocks). + +%% ------------------------------------------------------------------ +%% HIP 17 vars +%% +%% For every possible h3 resolution, we will define: +%% - number of siblings +%% - density_tgt +%% - density_max +%% +%% So hip17_res_0 value could be: 2, 10000, 10000 for example; +%% where num_siblings=2, density_tgt=10000, density_max=10000 +%% +%% We'd specify any of the below variables like so: <<"2,10000,10000">> +%% We expect the value of any of these variables to be in format: <<"int,int,int">> +-define(hip17_res_0, hip17_res_0). +-define(hip17_res_1, hip17_res_1). +-define(hip17_res_2, hip17_res_2). +-define(hip17_res_3, hip17_res_3). +-define(hip17_res_4, hip17_res_4). +-define(hip17_res_5, hip17_res_5). +-define(hip17_res_6, hip17_res_6). +-define(hip17_res_7, hip17_res_7). +-define(hip17_res_8, hip17_res_8). +-define(hip17_res_9, hip17_res_9). +-define(hip17_res_10, hip17_res_10). +-define(hip17_res_11, hip17_res_11). +-define(hip17_res_12, hip17_res_12). +-define(density_tgt_res, density_tgt_res). diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl new file mode 100644 index 0000000000..509c317b65 --- /dev/null +++ b/src/blockchain_hex.erl @@ -0,0 +1,217 @@ +-module(blockchain_hex). + +-export([var_map/1, scale/4, densities/1]). + +-include("blockchain_vars.hrl"). + +-type density_map() :: #{h3:h3_index() => pos_integer()}. +-type densities() :: {density_map(), density_map()}. +-type var_map() :: #{non_neg_integer() => map()}. +-type hex_resolutions() :: [non_neg_integer()]. +-type locations() :: [h3:h3_index()]. + +%%-------------------------------------------------------------------- +%% Public functions +%%-------------------------------------------------------------------- + +-spec densities(Ledger :: blockchain_ledger_v1:ledger()) -> densities(). +densities(Ledger) -> + %% build a map of required chain variables, example: + %% #{0 => #{n => N, density_tgt => T, density_max =M}, ....} + VarMap = var_map(Ledger), + %% Filter out gateways with no location + Locations = filtered_locations(Ledger), + %% Calculate clipped and unclipped densities + densities(VarMap, Locations). + +-spec scale( + Location :: h3:h3_index(), + TargetRes :: non_neg_integer(), + UnclippedDensities :: density_map(), + ClippedDensities :: density_map() +) -> float(). +scale(Location, TargetRes, UnclippedDensities, ClippedDensities) -> + lists:foldl( + fun(R, Acc) -> + Parent = h3:parent(Location, R), + Acc * (maps:get(Parent, ClippedDensities) / maps:get(Parent, UnclippedDensities)) + end, + 1.0, + lists:seq(TargetRes, 0, -1) + ). + +-spec var_map(Ledger :: blockchain_ledger_v1:ledger()) -> var_map(). +var_map(Ledger) -> + [N0, Tgt0, Max0] = get_density_var(?hip17_res_0, Ledger), + [N1, Tgt1, Max1] = get_density_var(?hip17_res_1, Ledger), + [N2, Tgt2, Max2] = get_density_var(?hip17_res_2, Ledger), + [N3, Tgt3, Max3] = get_density_var(?hip17_res_3, Ledger), + [N4, Tgt4, Max4] = get_density_var(?hip17_res_4, Ledger), + [N5, Tgt5, Max5] = get_density_var(?hip17_res_5, Ledger), + [N6, Tgt6, Max6] = get_density_var(?hip17_res_6, Ledger), + [N7, Tgt7, Max7] = get_density_var(?hip17_res_7, Ledger), + [N8, Tgt8, Max8] = get_density_var(?hip17_res_8, Ledger), + [N9, Tgt9, Max9] = get_density_var(?hip17_res_9, Ledger), + [N10, Tgt10, Max10] = get_density_var(?hip17_res_10, Ledger), + [N11, Tgt11, Max11] = get_density_var(?hip17_res_11, Ledger), + [N12, Tgt12, Max12] = get_density_var(?hip17_res_12, Ledger), + + #{ + 0 => #{n => N0, density_tgt => Tgt0, density_max => Max0}, + 1 => #{n => N1, density_tgt => Tgt1, density_max => Max1}, + 2 => #{n => N2, density_tgt => Tgt2, density_max => Max2}, + 3 => #{n => N3, density_tgt => Tgt3, density_max => Max3}, + 4 => #{n => N4, density_tgt => Tgt4, density_max => Max4}, + 5 => #{n => N5, density_tgt => Tgt5, density_max => Max5}, + 6 => #{n => N6, density_tgt => Tgt6, density_max => Max6}, + 7 => #{n => N7, density_tgt => Tgt7, density_max => Max7}, + 8 => #{n => N8, density_tgt => Tgt8, density_max => Max8}, + 9 => #{n => N9, density_tgt => Tgt9, density_max => Max9}, + 10 => #{n => N10, density_tgt => Tgt10, density_max => Max10}, + 11 => #{n => N11, density_tgt => Tgt11, density_max => Max11}, + 12 => #{n => N12, density_tgt => Tgt12, density_max => Max12} + }. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +-spec filtered_locations(Ledger :: blockchain_ledger_v1:ledger()) -> locations(). +filtered_locations(Ledger) -> + AG = blockchain_ledger_v1:active_gateways(Ledger), + UnfilteredLocs = [blockchain_ledger_gateway_v2:location(G) || G <- maps:values(AG)], + lists:filter(fun(L) -> L /= undefined end, UnfilteredLocs). + +-spec hex_resolutions(VarMap :: var_map()) -> hex_resolutions(). +hex_resolutions(VarMap) -> + %% [12, 11, ... 0] + lists:reverse(lists:sort(maps:keys(VarMap))). + +-spec densities(VarMap :: var_map(), Locations :: locations()) -> densities(). +densities(VarMap, Locations) -> + %% Head = 12, Tail = [11, 10, ... 0] + [Head | Tail] = hex_resolutions(VarMap), + + %% find parent hexs to all hotspots at highest resolution in chain variables + ParentHexes = [h3:parent(Hex, Head) || Hex <- Locations], + + InitialDensities = init_densities(ParentHexes, #{}), + + {UDensities, Densities} = build_densities( + VarMap, + ParentHexes, + {InitialDensities, InitialDensities}, + Tail + ), + + {UDensities, Densities}. + +-spec init_densities(ParentHexes :: locations(), Init :: density_map()) -> density_map(). +init_densities(ParentHexes, Init) -> + lists:foldl( + fun(Hex, Acc) -> + maps:update_with( + Hex, + fun(V) -> V + 1 end, + 1, + Acc + ) + end, + Init, + ParentHexes + ). + +-spec build_densities(var_map(), locations(), densities(), [non_neg_integer()]) -> densities(). +build_densities(_VarMap, _ParentHexes, {UAcc, Acc}, []) -> + {UAcc, Acc}; +build_densities(VarMap, ParentHexes, {UAcc, Acc}, [Res | Tail]) -> + ChildHexes = lists:usort(ParentHexes), + + ChildToParents = [{Hex, h3:parent(Hex, Res)} || Hex <- ChildHexes], + + {UM0, M0} = unclipped_densities(ChildToParents, {UAcc, Acc}), + + OccupiedHexesThisRes = lists:usort([ThisParentHex || {_, ThisParentHex} <- ChildToParents]), + + {UM1, M1} = lists:foldl( + fun(ThisResHex, {UAcc3, Acc3}) -> + OccupiedCount = occupied_count(Res, VarMap, ThisResHex, M0), + + Limit = limit(Res, VarMap, OccupiedCount), + + { + maps:put(ThisResHex, maps:get(ThisResHex, M0), UAcc3), + maps:put(ThisResHex, min(Limit, maps:get(ThisResHex, M0)), Acc3) + } + end, + {UM0, M0}, + OccupiedHexesThisRes + ), + + build_densities(VarMap, OccupiedHexesThisRes, {UM1, M1}, Tail). + +-spec limit( + Res :: non_neg_integer(), + VarMap :: var_map(), + OccupiedCount :: non_neg_integer() +) -> non_neg_integer(). +limit(Res, VarMap, OccupiedCount) -> + min( + maps:get(density_max, maps:get(Res, VarMap)), + maps:get(density_tgt, maps:get(Res, VarMap)) * + max((OccupiedCount - maps:get(n, maps:get(Res, VarMap))), 1) + ). + +-spec occupied_count( + Res :: non_neg_integer(), + VarMap :: var_map(), + ThisResHex :: h3:h3_index(), + DensityMap :: density_map() +) -> non_neg_integer(). +occupied_count(Res, VarMap, ThisResHex, DensityMap) -> + H3Neighbors = h3:k_ring(ThisResHex, 1), + lists:foldl( + fun(Neighbor, Acc) -> + ToAdd = + case + maps:get(Neighbor, DensityMap, 0) >= + maps:get(density_tgt, maps:get(Res, VarMap)) + of + false -> 0; + true -> 1 + end, + Acc + ToAdd + end, + 0, + H3Neighbors + ). + +-spec unclipped_densities(locations(), densities()) -> densities(). +unclipped_densities(ChildToParents, {UAcc, Acc}) -> + lists:foldl( + fun({ChildHex, ThisParentHex}, {UAcc2, Acc2}) -> + {maps:update_with( + ThisParentHex, + fun(V) -> V + maps:get(ChildHex, Acc, 0) end, + maps:get(ChildHex, Acc, 0), + UAcc2 + ), + maps:update_with( + ThisParentHex, + fun(V) -> V + maps:get(ChildHex, Acc, 0) end, + maps:get(ChildHex, Acc, 0), + Acc2 + )} + end, + {UAcc, Acc}, + ChildToParents + ). + +-spec get_density_var( + Var :: atom(), + Ledger :: blockchain_ledger_v1:ledger() +) -> [pos_integer()]. +get_density_var(Var, Ledger) -> + {ok, Bin} = blockchain:config(Var, Ledger), + [N, Tgt, Max] = [list_to_integer(I) || I <- string:tokens(binary:bin_to_list(Bin), ",")], + [N, Tgt, Max]. diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index 5ae1b6fcae..e1fbe44cef 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -250,10 +250,11 @@ get_rewards_for_epoch(Current, End, Chain, Vars, Ledger, ChallengerRewards, Chal true -> {error, already_existing_rewards}; false -> + {UnclippedDensities, ClippedDensities} = blockchain_hex:densities(Ledger), get_rewards_for_epoch(Current+1, End, Chain, Vars, Ledger, poc_challengers_rewards(Transactions, Vars, ChallengerRewards), - poc_challengees_rewards(Transactions, Vars, Chain, Ledger, ChallengeeRewards), - poc_witnesses_rewards(Transactions, Vars, Chain, Ledger, WitnessRewards), + poc_challengees_rewards(Transactions, Vars, Chain, Ledger, ChallengeeRewards, UnclippedDensities, ClippedDensities), + poc_witnesses_rewards(Transactions, Vars, Chain, Ledger, WitnessRewards, UnclippedDensities, ClippedDensities), dc_rewards(Transactions, End, Vars, Ledger, DCRewards)) end end. @@ -305,6 +306,11 @@ get_reward_vars(Start, End, Ledger) -> _ -> undefined end, + DensityTgtRes = case blockchain:config(?density_tgt_res, Ledger) of + {ok, D} -> D; + _ -> undefined + end, + EpochReward = calculate_epoch_reward(Start, End, Ledger), #{ monthly_reward => MonthlyReward, @@ -322,7 +328,8 @@ get_reward_vars(Start, End, Ledger) -> poc_version => POCVersion, reward_version => RewardVersion, witness_redundancy => WitnessRedundancy, - poc_reward_decay_rate => DecayRate + poc_reward_decay_rate => DecayRate, + density_tgt_res => DensityTgtRes }. -spec calculate_epoch_reward(pos_integer(), pos_integer(), blockchain_ledger_v1:ledger()) -> float(). @@ -456,12 +463,16 @@ normalize_challenger_rewards(ChallengerRewards, #{epoch_reward := EpochReward, Vars :: map(), Chain :: blockchain:blockchain(), Ledger :: blockchain_ledger_v1:ledger(), - map()) -> #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. + ExistingRewards :: map(), + UnclippedDensities :: blockchain_hex:density_map(), + ClippedDensities :: blockchain_hex:density_map()) -> #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. poc_challengees_rewards(Transactions, Vars, Chain, Ledger, - ExistingRewards) -> + ExistingRewards, + UnclippedDensities, + ClippedDensities) -> lists:foldl( fun(Txn, Acc0) -> case blockchain_txn:type(Txn) == blockchain_txn_poc_receipts_v1 of @@ -469,7 +480,7 @@ poc_challengees_rewards(Transactions, Acc0; true -> Path = blockchain_txn_poc_receipts_v1:path(Txn), - poc_challengees_rewards_(Vars, Path, Path, Txn, Chain, Ledger, true, Acc0) + poc_challengees_rewards_(Vars, Path, Path, Txn, Chain, Ledger, true, UnclippedDensities, ClippedDensities, Acc0) end end, ExistingRewards, @@ -493,7 +504,7 @@ normalize_challengee_rewards(ChallengeeRewards, #{epoch_reward := EpochReward, ChallengeeRewards ). -poc_challengees_rewards_(_Vars, [], _StaticPath, _Txn, _Chain, _Ledger, _, Acc) -> +poc_challengees_rewards_(_Vars, [], _StaticPath, _Txn, _Chain, _Ledger, _, _, _, Acc) -> Acc; poc_challengees_rewards_(#{poc_version := Version}=Vars, [Elem|Path], @@ -502,12 +513,17 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, Chain, Ledger, IsFirst, + UnclippedDensities, + ClippedDensities, Acc0) when Version >= 2 -> WitnessRedundancy = maps:get(witness_redundancy, Vars, undefined), DecayRate = maps:get(poc_reward_decay_rate, Vars, undefined), + DensityTgtRes = maps:get(density_tgt_res, Vars, undefined), %% check if there were any legitimate witnesses Witnesses = legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version), Challengee = blockchain_poc_path_element_v1:challengee(Elem), + {ok, ChallengeeGw} = blockchain_ledger_v1:find_gateway_info(Challengee, Ledger), + ChallengeeLoc = blockchain_ledger_gateway_v2:location(ChallengeeGw), I = maps:get(Challengee, Acc0, 0), case blockchain_poc_path_element_v1:receipt(Elem) of undefined -> @@ -524,7 +540,11 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+1, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(DensityTgtRes, + ChallengeeLoc, + UnclippedDensities, + ClippedDensities), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; true when is_integer(Version), Version > 4, IsFirst == false -> %% while we don't have a receipt for this node, we do know @@ -537,12 +557,16 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+2, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(DensityTgtRes, + ChallengeeLoc, + UnclippedDensities, + ClippedDensities), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; _ -> Acc0 end, - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, Acc1); + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, UnclippedDensities, ClippedDensities, Acc1); Receipt -> case blockchain_poc_receipt_v1:origin(Receipt) of radio -> @@ -559,7 +583,11 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+3, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(DensityTgtRes, + ChallengeeLoc, + UnclippedDensities, + ClippedDensities), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; false when is_integer(Version), Version > 4 -> %% this challengee rx'd and sent a receipt @@ -568,7 +596,11 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+2, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(DensityTgtRes, + ChallengeeLoc, + UnclippedDensities, + ClippedDensities), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; _ -> case poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) of @@ -576,10 +608,14 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+1, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(DensityTgtRes, + ChallengeeLoc, + UnclippedDensities, + ClippedDensities), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end end, - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, Acc1); + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, UnclippedDensities, ClippedDensities, Acc1); p2p -> %% if there are legitimate witnesses or the path continues %% the challengee did their job @@ -597,7 +633,11 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+2, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(DensityTgtRes, + ChallengeeLoc, + UnclippedDensities, + ClippedDensities), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; true -> case poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) of @@ -605,21 +645,25 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+1, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(DensityTgtRes, + ChallengeeLoc, + UnclippedDensities, + ClippedDensities), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end end, - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, Acc1) + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, UnclippedDensities, ClippedDensities, Acc1) end end; -poc_challengees_rewards_(Vars, [Elem|Path], StaticPath, Txn, Chain, Ledger, _IsFirst, Acc0) -> +poc_challengees_rewards_(Vars, [Elem|Path], StaticPath, Txn, Chain, Ledger, _IsFirst, UnclippedDensities, ClippedDensities, Acc0) -> case blockchain_poc_path_element_v1:receipt(Elem) of undefined -> - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, Acc0); + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, UnclippedDensities, ClippedDensities, Acc0); _Receipt -> Challengee = blockchain_poc_path_element_v1:challengee(Elem), I = maps:get(Challengee, Acc0, 0), Acc1 = maps:put(Challengee, I+1, Acc0), - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, Acc1) + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, UnclippedDensities, ClippedDensities, Acc1) end. -spec poc_challengee_reward_unit(WitnessRedundancy :: undefined | pos_integer(), @@ -640,14 +684,19 @@ poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) -> Vars :: map(), Chain :: blockchain:blockchain(), Ledger :: blockchain_ledger_v1:ledger(), - WitnessRewards :: map()) -> #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. + WitnessRewards :: map(), + UnclippedDensities :: blockchain_hex:density_map(), + ClippedDensities :: blockchain_hex:density_map()) -> #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. poc_witnesses_rewards(Transactions, #{poc_version := POCVersion}=Vars, Chain, Ledger, - WitnessRewards) -> + WitnessRewards, + UnclippedDensities, + ClippedDensities) -> WitnessRedundancy = maps:get(witness_redundancy, Vars, undefined), DecayRate = maps:get(poc_reward_decay_rate, Vars, undefined), + DensityTgtRes = maps:get(density_tgt_res, Vars, undefined), lists:foldl( fun(Txn, Acc0) -> case blockchain_txn:type(Txn) == blockchain_txn_poc_receipts_v1 of @@ -685,15 +734,36 @@ poc_witnesses_rewards(Transactions, U end, - lists:foldl( - fun(WitnessRecord, Map) -> - Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), - I = maps:get(Witness, Map, 0), - maps:put(Witness, I+ToAdd, Map) - end, - Acc1, - ValidWitnesses - ) + case DensityTgtRes of + undefined -> + %% old (HIP15) + lists:foldl( + fun(WitnessRecord, Map) -> + Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), + I = maps:get(Witness, Map, 0), + maps:put(Witness, I+ToAdd, Map) + end, + Acc1, + ValidWitnesses + ); + D -> + %% new (HIP17) + lists:foldl( + fun(WitnessRecord, Map) -> + Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), + {ok, WitnessGw} = blockchain_ledger_v1:find_gateway_info(Witness, Ledger), + WitnessLoc = blockchain_ledger_gateway_v2:location(WitnessGw), + RxScale = blockchain_hex:scale(WitnessLoc, + D, + UnclippedDensities, + ClippedDensities), + I = maps:get(Witness, Map, 0), + maps:put(Witness, I+(ToAdd*RxScale), Map) + end, + Acc1, + ValidWitnesses + ) + end end end, Acc0, @@ -1722,3 +1792,15 @@ common_poc_vars() -> }. -endif. + +maybe_calc_tx_scale(DensityTgtRes, + ChallengeeLoc, + UnclippedDensities, + ClippedDensities) -> + case DensityTgtRes of + undefined -> 1.0; + D -> blockchain_hex:scale(ChallengeeLoc, + D, + UnclippedDensities, + ClippedDensities) + end. diff --git a/src/transactions/v1/blockchain_txn_vars_v1.erl b/src/transactions/v1/blockchain_txn_vars_v1.erl index 8500ba8905..331dceba11 100644 --- a/src/transactions/v1/blockchain_txn_vars_v1.erl +++ b/src/transactions/v1/blockchain_txn_vars_v1.erl @@ -1071,10 +1071,100 @@ validate_var(?use_multi_keys, Value) -> validate_var(?transfer_hotspot_stale_poc_blocks, Value) -> validate_int(Value, "transfer_hotspot_stale_poc_blocks", 1, 50000, false); +%% HIP 17 vars +validate_var(?hip17_res_0, Value) -> + validate_hip17_vars(Value, "hip17_res_0"); +validate_var(?hip17_res_1, Value) -> + validate_hip17_vars(Value, "hip17_res_1"); +validate_var(?hip17_res_2, Value) -> + validate_hip17_vars(Value, "hip17_res_2"); +validate_var(?hip17_res_3, Value) -> + validate_hip17_vars(Value, "hip17_res_3"); +validate_var(?hip17_res_4, Value) -> + validate_hip17_vars(Value, "hip17_res_4"); +validate_var(?hip17_res_5, Value) -> + validate_hip17_vars(Value, "hip17_res_5"); +validate_var(?hip17_res_6, Value) -> + validate_hip17_vars(Value, "hip17_res_6"); +validate_var(?hip17_res_7, Value) -> + validate_hip17_vars(Value, "hip17_res_7"); +validate_var(?hip17_res_8, Value) -> + validate_hip17_vars(Value, "hip17_res_8"); +validate_var(?hip17_res_9, Value) -> + validate_hip17_vars(Value, "hip17_res_9"); +validate_var(?hip17_res_10, Value) -> + validate_hip17_vars(Value, "hip17_res_10"); +validate_var(?hip17_res_11, Value) -> + validate_hip17_vars(Value, "hip17_res_11"); +validate_var(?hip17_res_12, Value) -> + validate_hip17_vars(Value, "hip17_res_12"); +validate_var(?density_tgt_res, Value) -> + validate_int(Value, "density_tgt_res", 1, 15, false); + validate_var(Var, Value) -> %% something we don't understand, crash invalid_var(Var, Value). +validate_hip17_vars(Value, Var) when is_binary(Value) -> + %% We expect the value of the variable to be in format: <<"int,int,int">> + case size(Value) of + 3 -> + case get_density_var(Value) of + {error, _}=E0 -> + lager:error("unable to get densit var, reason: ~p", [E0]), + throw({error, {invalid_density_var, Var, Value}}); + {ok, Res} -> + [Siblings, DensityTgt, DensityMax] = Res, + CheckSiblings = validate_int_min_max(Siblings, "siblings", 1, 1000), + CheckDensityTgt = validate_int_min_max(DensityTgt, "density_tgt", 1, 200000), + CheckDensityMax = validate_int_min_max(DensityMax, "density_max", 1, 200000), + + case CheckSiblings of + {error, _}=E1 -> + lager:error("invalid_siblings, reason: ~p", [E1]), + throw({error, {invalid_siblings, Var, Value}}); + ok -> + case CheckDensityTgt of + {error, _}=E2 -> + lager:error("invalid_density_tgt, reason: ~p", [E2]), + throw({error, {invalid_density_tgt, Var, Value}}); + ok -> + case CheckDensityMax of + {error, _}=E3 -> + lager:error("invalid_density_max, reason: ~p", [E3]), + throw({error, {invalid_density_max, Var, Value}}); + ok -> + ok + end + end + end + end; + _ -> + throw({error, {invalid_size, Var, Value}}) + end; +validate_hip17_vars(Value, Var) -> + throw({error, {invalid_format, Var, Value}}). + +validate_int_min_max(Value, Name, Min, Max) -> + case Value >= Min andalso Value =< Max of + false -> {error, {list_to_atom(Name ++ "_out_of_range"), Value}}; + _ -> ok + end. + +get_density_var(Value) -> + try + Res = [list_to_integer(I) || I <- string:tokens(binary:bin_to_list(Value), ",")], + {ok, Res} + catch + What:Why:Stack -> + lager:error("Unable to get_density_var, What: ~p, Why: ~p, Stack: ~p", [ + What, + Why, + Stack + ]), + {error, {unable_to_get_density_var, Value}} + end. + -ifdef(TEST). invalid_var(Var, Value) -> case lists:member(Var, ?exceptions) of % test only diff --git a/test/blockchain_ct_utils.erl b/test/blockchain_ct_utils.erl index 0fbead0cbe..7d4a74ab58 100644 --- a/test/blockchain_ct_utils.erl +++ b/test/blockchain_ct_utils.erl @@ -24,7 +24,9 @@ create_vars/0, create_vars/1, raw_vars/1, init_base_dir_config/3, - join_packet/3 + join_packet/3, + ledger/2, + destroy_ledger/0 ]). pmap(F, L) -> @@ -431,3 +433,73 @@ reverse_bin(Bin) -> reverse_bin(Bin, <<>>). reverse_bin(<<>>, Acc) -> Acc; reverse_bin(<>, Acc) -> reverse_bin(Rest, <>). + +ledger(ExtraVars, S3URL) -> + %% Ledger at height: 481929 + %% ActiveGateway Count: 8000 + {ok, Dir} = file:get_cwd(), + %% Ensure priv dir exists + PrivDir = filename:join([Dir, "priv"]), + ok = filelib:ensure_dir(PrivDir ++ "/"), + %% Path to static ledger tar + LedgerTar = filename:join([PrivDir, "ledger.tar.gz"]), + %% Extract ledger tar if required + ok = extract_ledger_tar(PrivDir, LedgerTar, S3URL), + %% Get the ledger + Ledger = blockchain_ledger_v1:new(PrivDir), + %% Get current ledger vars + LedgerVars = ledger_vars(Ledger), + %% Ensure the ledger has the vars we're testing against + Ledger1 = blockchain_ledger_v1:new_context(Ledger), + blockchain_ledger_v1:vars(maps:merge(LedgerVars, ExtraVars), [], Ledger1), + %% If the hexes aren't on the ledger add them + blockchain:bootstrap_hexes(Ledger1), + blockchain_ledger_v1:commit_context(Ledger1), + Ledger. + +extract_ledger_tar(PrivDir, LedgerTar, S3URL) -> + case filelib:is_file(LedgerTar) of + true -> + %% if we have already unpacked it, no need to do it again + LedgerDB = filename:join([PrivDir, "ledger.db"]), + case filelib:is_dir(LedgerDB) of + true -> + ok; + false -> + %% ledger tar file present, extract + erl_tar:extract(LedgerTar, [compressed, {cwd, PrivDir}]) + end; + false -> + %% ledger tar file not found, download & extract + ok = ssl:start(), + {ok, {{_, 200, "OK"}, _, Body}} = httpc:request(S3URL), + ok = file:write_file(filename:join([PrivDir, "ledger.tar.gz"]), Body), + erl_tar:extract(LedgerTar, [compressed, {cwd, PrivDir}]) + end. + +ledger_vars(Ledger) -> + blockchain_utils:vars_binary_keys_to_atoms(maps:from_list(blockchain_ledger_v1:snapshot_vars(Ledger))). + +destroy_ledger() -> + {ok, Dir} = file:get_cwd(), + %% Ensure priv dir exists + PrivDir = filename:join([Dir, "priv"]), + ok = filelib:ensure_dir(PrivDir ++ "/"), + LedgerTar = filename:join([PrivDir, "ledger.tar.gz"]), + LedgerDB = filename:join([PrivDir, "ledger.db"]), + + case filelib:is_file(LedgerTar) of + true -> + %% we found a ledger tarball, remove it + file:delete(LedgerTar); + false -> + ok + end, + case filelib:is_dir(LedgerDB) of + true -> + %% we found a ledger.db, remove it + file:del_dir(LedgerDB); + false -> + %% ledger.db dir not found, don't do anything + ok + end. diff --git a/test/blockchain_hex_SUITE.erl b/test/blockchain_hex_SUITE.erl new file mode 100644 index 0000000000..801fdccdc3 --- /dev/null +++ b/test/blockchain_hex_SUITE.erl @@ -0,0 +1,241 @@ +-module(blockchain_hex_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-include("blockchain_vars.hrl"). + +-export([ + all/0, + init_per_testcase/2, + end_per_testcase/2, + init_per_suite/1, + end_per_suite/1 +]). + +-export([ + non_zero_test/1, + known_values_test/1, + known_differences_test/1, + scale_test/1 +]). + +%% Values taken from python model +-define(KNOWN, [ + {631210968849144319, 1}, + {631179325713598463, 2}, + {631196205757572607, 1}, + {631243921328349695, 1}, + {631243921527814655, 3}, + {631211399470291455, 1}, + {577234808489377791, 2319}, + {577023702256844799, 449}, + {577692205326532607, 1039}, + {577340361605644287, 6}, + {576812596024311807, 54}, + {577621836582354943, 5}, + {577164439745200127, 1459}, + {579768083279773695, 2}, + {577305177233555455, 5}, + {576601489791778815, 9}, + {576636674163867647, 56}, + {576777411652222975, 3}, + {577481099093999615, 43}, + {578114417791598591, 1}, + {577269992861466623, 1}, + {577586652210266111, 24}, + {577762574070710271, 381}, + {577199624117288959, 1817}, + {577058886628933631, 1}, + {578325524024131583, 1}, + {577832942814887935, 26}, + {576953333512667135, 2}, + {579838452023951359, 1}, + {579451423930974207, 1}, + {576918149140578303, 600}, + {599663374669709311, 3}, + {599238598109167615, 1}, + {599653418935517183, 4}, + {600256126327455743, 2}, + {600176343014965247, 1}, + {599718760420474879, 2}, + {599736317173039103, 1}, + {599721183855771647, 2}, + {599685859897245695, 1} +]). + +%% There are 1952 differences when we calculate clipped vs unclipped densities +-define(KNOWN_CLIP_VS_UNCLIPPED, 1952). + +%% Some known differences between clipped vs unclipped densities +-define(KNOWN_DIFFERENCES, #{ + 617712130192310271 => {1, 2}, + 617761313717485567 => {1, 2}, + 617733151106007039 => {2, 3}, + 617684909104562175 => {2, 3}, + 617700174986739711 => {1, 2}, + 617700552987901951 => {2, 4}, + 617700552891170815 => {2, 4}, + 617733123580887039 => {1, 2}, + 617733151091589119 => {1, 3}, + 617743888390553599 => {1, 3}, + 617733269885550591 => {1, 2} +}). + +%% Value taken from hip17 +-define(KNOWN_RES_FOR_SCALING, "8828361563fffff"). + +all() -> + [ + non_zero_test, + known_values_test, + known_differences_test, + scale_test + ]. + +%%-------------------------------------------------------------------- +%% TEST SUITE SETUP +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + LedgerURL = "https://blockchain-core.s3-us-west-1.amazonaws.com/ledger-586724.tar.gz", + Ledger = blockchain_ct_utils:ledger(hip17_vars(), LedgerURL), + + %% Check that the pinned ledger is at the height we expect it to be + {ok, 586724} = blockchain_ledger_v1:current_height(Ledger), + + %% Check that the vars are correct, one is enough... + VarMap = blockchain_hex:var_map(Ledger), + Res4 = maps:get(4, VarMap), + 1 = maps:get(n, Res4), + 250 = maps:get(density_tgt, Res4), + 800 = maps:get(density_max, Res4), + + {Time, {UnclippedDensities, ClippedDensities}} = timer:tc( + fun() -> + blockchain_hex:densities(Ledger) + end + ), + ct:pal("density calculation time: ~pms", [Time / 1000]), + + %% Check that the time is less than 1000ms + ?assert(1000 =< Time), + + [ + {ledger, Ledger}, + {clipped, ClippedDensities}, + {unclipped, UnclippedDensities}, + {var_map, VarMap} + | Config + ]. + +%%-------------------------------------------------------------------- +%% TEST CASE SETUP +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% TEST CASE TEARDOWN +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% TEST SUITE TEARDOWN +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + %% destroy the downloaded ledger + blockchain_ct_utils:destroy_ledger(), + ok. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +non_zero_test(Config) -> + ClippedDensities = ?config(clipped, Config), + + %% assert that there are no 0 values for density + %% we count a hotspot at res 12 (max resolution) as 1 + true = lists:all(fun(V) -> V /= 0 end, maps:values(ClippedDensities)), + ok. + +known_values_test(Config) -> + ClippedDensities = ?config(clipped, Config), + + %% assert some known values calculated from the python model (thanks @para1!) + true = lists:all( + fun({Hex, Density}) -> + Density == maps:get(Hex, ClippedDensities) + end, + ?KNOWN + ), + + ok. + +known_differences_test(Config) -> + UnclippedDensities = ?config(unclipped, Config), + ClippedDensities = ?config(clipped, Config), + + Differences = lists:foldl( + fun({Hex, ClippedDensity}, Acc) -> + UnclippedDensity = maps:get(Hex, UnclippedDensities), + case ClippedDensity /= UnclippedDensity of + false -> + Acc; + true -> + maps:put(Hex, {ClippedDensity, UnclippedDensity}, Acc) + end + end, + #{}, + maps:to_list(ClippedDensities) + ), + + ?assertEqual(?KNOWN_CLIP_VS_UNCLIPPED, map_size(Differences)), + + true = lists:all( + fun({Hex, {Clipped, Unclipped}}) -> + Unclipped == maps:get(Hex, UnclippedDensities) andalso + Clipped == maps:get(Hex, ClippedDensities) + end, + maps:to_list(?KNOWN_DIFFERENCES) + ), + + ok. + +scale_test(Config) -> + UnclippedDensities = ?config(unclipped, Config), + ClippedDensities = ?config(clipped, Config), + + Another = h3:from_string("8c2836152804dff"), + + ok = lists:foreach(fun(I) -> + Scale = blockchain_hex:scale(Another, I, UnclippedDensities, ClippedDensities), + ct:pal("Res: ~p, Scale: ~p", [I, Scale]) + end, lists:seq(12, 0, -1)), + + %% TODO: Assert checks from the python model + + ok. + +%%-------------------------------------------------------------------- +%% CHAIN VARIABLES +%%-------------------------------------------------------------------- + +hip17_vars() -> + #{ + hip17_res_0 => <<"2,100000,100000">>, + hip17_res_1 => <<"2,100000,100000">>, + hip17_res_2 => <<"2,100000,100000">>, + hip17_res_3 => <<"2,100000,100000">>, + hip17_res_4 => <<"1,250,800">>, + hip17_res_5 => <<"1,100,400">>, + hip17_res_6 => <<"1,25,100">>, + hip17_res_7 => <<"2,5,20">>, + hip17_res_8 => <<"2,1,4">>, + hip17_res_9 => <<"2,1,2">>, + hip17_res_10 => <<"2,1,1">>, + hip17_res_11 => <<"2,100000,100000">>, + hip17_res_12 => <<"2,100000,100000">>, + density_tgt_res => 8 + }. From 6b0335df95ed4b7bfca45a1ae97c1ed5f8d3352f Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Fri, 20 Nov 2020 11:33:10 -0600 Subject: [PATCH 02/18] Add h3dex - Calculate k-nearest hex; plumb assert update - Correct some misunderstandings - Add h3 to key EQC test --- eqc/h3dex_eqc.erl | 54 ++++++++ rebar.config | 2 +- rebar.lock | 2 +- src/blockchain.erl | 24 +++- src/ledger/v1/blockchain_ledger_v1.erl | 120 ++++++++++++++++-- .../v1/blockchain_txn_assert_location_v1.erl | 7 +- 6 files changed, 196 insertions(+), 13 deletions(-) create mode 100644 eqc/h3dex_eqc.erl diff --git a/eqc/h3dex_eqc.erl b/eqc/h3dex_eqc.erl new file mode 100644 index 0000000000..707c0c0300 --- /dev/null +++ b/eqc/h3dex_eqc.erl @@ -0,0 +1,54 @@ +-module(h3dex_eqc). + +-include_lib("eqc/include/eqc.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(INDICES, lists:flatten([ h3:children(I, 5) || I <- h3:get_res0_indexes()])). + +-export([prop_h3dex_check/0]). + +prop_h3dex_check() -> + ?FORALL({Hex1, Hex2, Resolution}, ?SUCHTHAT({X1, X2, _}, {gen_h3(), gen_h3(), choose(6, 6)}, X1 /= 0 andalso X2 /= 0 andalso X1 /= X2), + begin + Children1 = h3:children(Hex1, Resolution) -- [0], + Children2 = h3:children(Hex2, Resolution) -- [0], + + End = h3_to_key(Hex1), + Start = find_lower_bound_hex(Hex1), + ?WHENFAIL(begin + io:format("Hex 1: ~p (~w -> ~w), Hex 2: ~p (~w -> ~w), Resolution ~p~n", [Hex1, find_lower_bound_hex(Hex1), h3_to_key(Hex1), Hex2, find_lower_bound_hex(Hex2), h3_to_key(Hex2), Resolution]) + end, + noshrink(conjunction( + [{all_children_in_range, eqc:equals([], lists:filter(fun(X) -> X < Start orelse X > End end, lists:sort(lists:map(fun h3_to_key/1, Children1)))) }, + {all_non_children_out_of_range, eqc:equals([], lists:filter(fun(E) -> X = h3_to_key(E), X > Start andalso X =< End end, Children2)) }, + {unparse_works, eqc:equals(Hex1, key_to_h3(h3_to_key(Hex1)))} + ] + ) + ) + ) + end). + +gen_h3() -> + elements(?INDICES). + +h3_to_key(H3) -> + %% both reserved fields must be 0 and Mode must be 1 for this to be a h3 cell + <<0:1/integer-unsigned-big, 1:4/integer-unsigned-big, 0:3/integer-unsigned-big, Resolution:4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits:45/integer-unsigned-big>> = <>, + %% store the resolution inverted (15 - Resolution) so it sorts later + <>. + +key_to_h3(Key) -> + <> = Key, + <> = <<0:1, 1:4/integer-unsigned-big, 0:3, (15 - InverseResolution):4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits:45/integer-unsigned-big>>, + H3. + +find_lower_bound_hex(Hex) -> + %% both reserved fields must be 0 and Mode must be 1 for this to be a h3 cell + <<0:1, 1:4/integer-unsigned-big, 0:3, Resolution:4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits/bitstring>> = <>, + ActualDigitCount = Resolution * 3, + %% pull out the actual digits used and dump the rest + <> = Digits, + Padding = 45 - ActualDigitCount, + %% store the resolution inverted (15 - 15) = 0 so it sorts earlier + %% pad the actual digits used with 0s on the end + <>. diff --git a/rebar.config b/rebar.config index 682b77eb21..ed788ac8e0 100644 --- a/rebar.config +++ b/rebar.config @@ -34,7 +34,7 @@ {base64url, "1.0.1"}, {libp2p, ".*", {git, "https://github.com/helium/erlang-libp2p.git", {branch, "master"}}}, {clique, ".*", {git, "https://github.com/helium/clique.git", {branch, "develop"}}}, - {h3, ".*", {git, "https://github.com/helium/erlang-h3.git", {branch, "master"}}}, + {h3, ".*", {git, "https://github.com/helium/erlang-h3.git", {branch, "adt/res0_indexes"}}}, {erl_angry_purple_tiger, ".*", {git, "https://github.com/helium/erl_angry_purple_tiger.git", {branch, "master"}}}, {erlang_stats, ".*", {git, "https://github.com/helium/erlang-stats.git", {branch, "master"}}}, {e2qc, ".*", {git, "https://github.com/project-fifo/e2qc", {branch, "master"}}}, diff --git a/rebar.lock b/rebar.lock index 831e509eaa..b4bd23201b 100644 --- a/rebar.lock +++ b/rebar.lock @@ -37,7 +37,7 @@ {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1}, {<<"h3">>, {git,"https://github.com/helium/erlang-h3.git", - {ref,"a92737698d45c97b7b9b6694513b48c29522a42e"}}, + {ref,"040cf9226699e0a1f85b14cad4dd924e5618578e"}}, 0}, {<<"helium_proto">>, {git,"https://github.com/helium/proto.git", diff --git a/src/blockchain.erl b/src/blockchain.erl index 1a1aa7a904..32fc7b5a9c 100644 --- a/src/blockchain.erl +++ b/src/blockchain.erl @@ -94,7 +94,8 @@ -define(BC_UPGRADE_FUNS, [fun upgrade_gateways_v2/1, fun bootstrap_hexes/1, - fun upgrade_gateways_oui/1]). + fun upgrade_gateways_oui/1, + fun bootstrap_h3dex/1]). -type blocks() :: #{blockchain_block:hash() => blockchain_block:block()}. -type blockchain() :: #blockchain{}. @@ -262,6 +263,27 @@ upgrade_gateways_oui_(Ledger) -> end, Gateways), ok. +-spec bootstrap_h3dex(blockchain_ledger_v1:ledger()) -> ok. +%% @doc Bootstrap the H3Dex for both the active and delayed ledgers +bootstrap_h3dex(Ledger) -> + ok = do_bootstrap_h3dex(Ledger), + Ledger1 = blockchain_ledger_v1:mode(delayed, Ledger), + Ledger2 = blockchain_ledger_v1:new_context(Ledger1), + ok = do_bootstrap_h3dex(Ledger2), + blockchain_ledger_v1:commit_context(Ledger2). + +do_bootstrap_h3dex(Ledger) -> + Gateways = blockchain_ledger_v1:active_gateways(Ledger), + H3Dex = maps:fold( + fun(GwAddr, GW, Acc) -> + case blockchain_ledger_gateway_v2:location(GW) of + undefined -> Acc; + Location -> + maps:update_with(Location, fun(V) -> [GwAddr | V] end, [GwAddr], Acc) + end + end, #{}, Gateways), + blockchain_ledger_v1:set_h3dex(H3Dex, Ledger). + %%-------------------------------------------------------------------- %% @doc %% @end diff --git a/src/ledger/v1/blockchain_ledger_v1.erl b/src/ledger/v1/blockchain_ledger_v1.erl index 4e9e81d988..062ab20aef 100644 --- a/src/ledger/v1/blockchain_ledger_v1.erl +++ b/src/ledger/v1/blockchain_ledger_v1.erl @@ -120,6 +120,11 @@ clean_all_hexes/1, + set_h3dex/2, + lookup_gateways_from_hex/2, + add_gw_to_hex/3, + remove_gw_from_hex/3, + %% snapshot save/restore stuff snapshot_vars/1, @@ -142,6 +147,8 @@ load_state_channels/2, snapshot_hexes/1, load_hexes/2, + snapshot_h3dex/1, + load_h3dex/2, snapshot_delayed_vars/1, load_delayed_vars/2, snapshot_threshold_txns/1, @@ -207,6 +214,7 @@ routing :: rocksdb:cf_handle(), subnets :: rocksdb:cf_handle(), state_channels :: rocksdb:cf_handle(), + h3dex :: rocksdb:cf_handle(), cache :: undefined | ets:tid(), gateway_cache :: undefined | ets:tid() }). @@ -244,15 +252,16 @@ -type state_channel_map() :: #{blockchain_state_channel_v1:id() => blockchain_ledger_state_channel_v1:state_channel() | blockchain_ledger_state_channel_v2:state_channel_v2()}. - +-type h3dex() :: #{h3:h3_index() => [libp2p_crypto:pubkey_bin()]}. %% these keys are gateway addresses -export_type([ledger/0]). -spec new(file:filename_all()) -> ledger(). new(Dir) -> {ok, DB, CFs} = open_db(Dir), [DefaultCF, AGwsCF, EntriesCF, DCEntriesCF, HTLCsCF, PoCsCF, SecuritiesCF, RoutingCF, - SubnetsCF, SCsCF, DelayedDefaultCF, DelayedAGwsCF, DelayedEntriesCF, DelayedDCEntriesCF, - DelayedHTLCsCF, DelayedPoCsCF, DelayedSecuritiesCF, DelayedRoutingCF, DelayedSubnetsCF, DelayedSCsCF] = CFs, + SubnetsCF, SCsCF, H3DexCF, DelayedDefaultCF, DelayedAGwsCF, DelayedEntriesCF, + DelayedDCEntriesCF, DelayedHTLCsCF, DelayedPoCsCF, DelayedSecuritiesCF, + DelayedRoutingCF, DelayedSubnetsCF, DelayedSCsCF, DelayedH3DexCF] = CFs, #ledger_v1{ dir=Dir, db=DB, @@ -268,7 +277,8 @@ new(Dir) -> securities=SecuritiesCF, routing=RoutingCF, subnets=SubnetsCF, - state_channels=SCsCF + state_channels=SCsCF, + h3dex=H3DexCF }, delayed= #sub_ledger_v1{ default=DelayedDefaultCF, @@ -280,7 +290,8 @@ new(Dir) -> securities=DelayedSecuritiesCF, routing=DelayedRoutingCF, subnets=DelayedSubnetsCF, - state_channels=DelayedSCsCF + state_channels=DelayedSCsCF, + h3dex=DelayedH3DexCF } }. @@ -2795,6 +2806,10 @@ state_channels_cf(#ledger_v1{mode=active, active=#sub_ledger_v1{state_channels=S state_channels_cf(#ledger_v1{mode=delayed, delayed=#sub_ledger_v1{state_channels=SCsCF}}) -> SCsCF. +-spec h3dex_cf(ledger()) -> rocksdb:cf_handle(). +h3dex_cf(#ledger_v1{mode=active, active=#sub_ledger_v1{h3dex=H3DexCF}}) -> H3DexCF; +h3dex_cf(#ledger_v1{mode=delayed, delayed=#sub_ledger_v1{h3dex=H3DexCF}}) -> H3DexCF. + -spec cache_put(ledger(), rocksdb:cf_handle(), binary(), binary()) -> ok. cache_put(Ledger, CF, Key, Value) -> {Cache, _GwCache} = context_cache(Ledger), @@ -2950,9 +2965,12 @@ open_db(Dir) -> CFOpts = GlobalOpts, - DefaultCFs = ["default", "active_gateways", "entries", "dc_entries", "htlcs", "pocs", "securities", "routing", "subnets", "state_channels", - "delayed_default", "delayed_active_gateways", "delayed_entries", "delayed_dc_entries", "delayed_htlcs", - "delayed_pocs", "delayed_securities", "delayed_routing", "delayed_subnets","delayed_state_channels"], + DefaultCFs = ["default", "active_gateways", "entries", "dc_entries", "htlcs", + "pocs", "securities", "routing", "subnets", "state_channels", "h3dex", + "delayed_default", "delayed_active_gateways", "delayed_entries", + "delayed_dc_entries", "delayed_htlcs", "delayed_pocs", + "delayed_securities", "delayed_routing", "delayed_subnets", + "delayed_state_channels", "delayed_h3dex"], ExistingCFs = case rocksdb:list_column_families(DBDir, DBOptions) of {ok, CFs0} -> @@ -3090,6 +3108,78 @@ clean_all_hexes(Ledger) -> _ -> ok end. +-spec set_h3dex(h3dex(), ledger()) -> ok. +set_h3dex(H3Dex, Ledger) -> + H3CF = h3dex_cf(Ledger), + _ = maps:map(fun(Loc, Gateways) -> + BinLoc = <>, + BinGWs = term_to_binary(Gateways, [compressed]), + cache_put(Ledger, H3CF, BinLoc, BinGWs) + end, H3Dex), + ok. + +-spec lookup_gateways_from_hex(Hex :: non_neg_integer(), + Ledger :: ledger()) -> {ok, Results :: h3dex()}. +%% @doc Given a hex find candidate gateways in the span to the next adjacent +%% hex. N.B. May return an empty map. +lookup_gateways_from_hex(Hex, Ledger) -> + H3CF = h3dex_cf(Ledger), + Res = cache_fold(Ledger, H3CF, + fun({<>, GWs}, Acc) -> + maps:put(Loc, binary_to_term(GWs), Acc) + end, #{}, [ + {start, find_lower_bound_hex(Hex)}, + {iterate_upper_bound, parse_h3(Hex)} + ] + ), + {ok, Res}. + +-spec find_lower_bound_hex(Hex :: non_neg_integer()) -> binary(). +%% @doc Let's find the nearest set of k neighbors for this hex at the +%% same resolution and return the "lowest" one. Since these numbers +%% are actually packed binaries, we will destructure them to sort better +%% lexically. +find_lower_bound_hex(Hex) -> + parse_h3(hd(h3:children(Hex, 15))). + +parse_h3(H3) -> + <<_Reserved:1, _Mode:4, _Reserved2:3, + Resolution:4, BaseCell:7, Digits:45>> = <>, + <>. + +-spec add_gw_to_hex(Hex :: non_neg_integer(), + GWAddr :: libp2p_crypto:pubkey_bin(), + Ledger :: ledger()) -> ok | {error, any()}. +%% @doc During an assert, this function will add a gateway address to a hex +add_gw_to_hex(Hex, GWAddr, Ledger) -> + H3CF = h3dex_cf(Ledger), + BinHex = <>, + case cache_get(Ledger, H3CF, BinHex, []) of + not_found -> + cache_put(Ledger, H3CF, BinHex, term_to_binary([GWAddr], [compressed])); + {ok, BinGws} -> + GWs = binary_to_term(BinGws), + cache_put(Ledger, H3CF, BinHex, term_to_binary([GWAddr | GWs], [compressed])); + Error -> Error + end. + +-spec remove_gw_from_hex(Hex :: non_neg_integer(), + GWAddr :: libp2p_crypto:pubkey_bin(), + Ledger :: ledger()) -> ok | {error, any()}. +%% @doc During an assert, if a gateway already had an asserted location +%% (and has been reasserted), this function will remove a gateway +%% address from a hex +remove_gw_from_hex(Hex, GWAddr, Ledger) -> + H3CF = h3dex_cf(Ledger), + BinHex = <>, + case cache_get(Ledger, H3CF, BinHex, []) of + not_found -> ok; + {ok, BinGws} -> + NewGWs = lists:delete(GWAddr, binary_to_term(BinGws)), + cache_put(Ledger, H3CF, BinHex, term_to_binary(NewGWs, [compressed])); + Error -> Error + end. + batch_from_cache(ETS) -> {ok, Batch} = rocksdb:batch(), ets:foldl(fun({{CF, Key}, ?CACHE_TOMBSTONE}, Acc) -> @@ -3441,6 +3531,20 @@ load_hexes(Hexes0, Ledger) -> ok end. +snapshot_h3dex(Ledger) -> + H3CF = h3dex_cf(Ledger), + lists:sort( + maps:to_list( + cache_fold( + Ledger, H3CF, + fun({Loc, GWs}, Acc) -> + maps:put(<>, binary_to_term(GWs), Acc) + end, #{}, + []))). + +load_h3dex(H3DexList, Ledger) -> + set_h3dex(maps:from_list(H3DexList), Ledger). + -spec get_sc_mod( Entry :: blockchain_ledger_state_channel_v1:state_channel() | blockchain_ledger_state_channel_v2:state_channel_v2(), Ledger :: ledger() ) -> blockchain_ledger_state_channel_v1 diff --git a/src/transactions/v1/blockchain_txn_assert_location_v1.erl b/src/transactions/v1/blockchain_txn_assert_location_v1.erl index b74ae74284..48b826bd3d 100644 --- a/src/transactions/v1/blockchain_txn_assert_location_v1.erl +++ b/src/transactions/v1/blockchain_txn_assert_location_v1.erl @@ -396,10 +396,13 @@ absorb(Txn, Chain) -> %% moved within the hex, no need to update ok; _ when OldHex == undefined -> - blockchain_ledger_v1:add_to_hex(Hex, Gateway, Ledger); + blockchain_ledger_v1:add_to_hex(Hex, Gateway, Ledger), + blockchain_ledger_v1:add_gw_to_hex(Hex, Gateway, Ledger); _ -> blockchain_ledger_v1:remove_from_hex(OldHex, Gateway, Ledger), - blockchain_ledger_v1:add_to_hex(Hex, Gateway, Ledger) + blockchain_ledger_v1:remove_gw_from_hex(OldHex, Gateway, Ledger), + blockchain_ledger_v1:add_to_hex(Hex, Gateway, Ledger), + blockchain_ledger_v1:add_gw_to_hex(Hex, Gateway, Ledger) end, From c38ed194dc34ce9d8b3bbe44d05c7357e5680b3b Mon Sep 17 00:00:00 2001 From: Andrew Thompson Date: Mon, 23 Nov 2020 07:03:11 -0800 Subject: [PATCH 03/18] Several improvements overall - Don't use active_gateways, use a fold - Use tested h3 keying scheme - Fix some other minor issues - Fix scale test - Switch to using h3dex for density map - More efficent initialization of densities - Remove nested loops - Fix eunits to use updated reward fun - Fix one more broken eunit and specs - Fix seek tuple and allow +1 upper bound - Add a simple known h3dex test - Review comment: Verbose spec for densities - Fix broken hip17 var validation --- src/blockchain.erl | 13 +- src/blockchain_hex.erl | 120 ++++++++---------- src/ledger/v1/blockchain_ledger_v1.erl | 88 ++++++++++--- .../v1/blockchain_txn_rewards_v1.erl | 53 ++++---- .../v1/blockchain_txn_vars_v1.erl | 21 ++- test/blockchain_hex_SUITE.erl | 37 +++++- 6 files changed, 193 insertions(+), 139 deletions(-) diff --git a/src/blockchain.erl b/src/blockchain.erl index 32fc7b5a9c..2afabee034 100644 --- a/src/blockchain.erl +++ b/src/blockchain.erl @@ -63,7 +63,7 @@ -include("blockchain_vars.hrl"). -ifdef(TEST). --export([bootstrap_hexes/1, can_add_block/2, get_plausible_blocks/1]). +-export([bootstrap_hexes/1, can_add_block/2, get_plausible_blocks/1, bootstrap_h3dex/1]). %% export a macro so we can interpose block saving to test failure -define(save_block(Block, Chain), ?MODULE:save_block(Block, Chain)). -include_lib("eunit/include/eunit.hrl"). @@ -273,16 +273,7 @@ bootstrap_h3dex(Ledger) -> blockchain_ledger_v1:commit_context(Ledger2). do_bootstrap_h3dex(Ledger) -> - Gateways = blockchain_ledger_v1:active_gateways(Ledger), - H3Dex = maps:fold( - fun(GwAddr, GW, Acc) -> - case blockchain_ledger_gateway_v2:location(GW) of - undefined -> Acc; - Location -> - maps:update_with(Location, fun(V) -> [GwAddr | V] end, [GwAddr], Acc) - end - end, #{}, Gateways), - blockchain_ledger_v1:set_h3dex(H3Dex, Ledger). + blockchain_ledger_v1:bootstrap_h3dex(Ledger). %%-------------------------------------------------------------------- %% @doc diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl index 509c317b65..4060cae020 100644 --- a/src/blockchain_hex.erl +++ b/src/blockchain_hex.erl @@ -5,10 +5,11 @@ -include("blockchain_vars.hrl"). -type density_map() :: #{h3:h3_index() => pos_integer()}. --type densities() :: {density_map(), density_map()}. +-type densities() :: {UnclippedDensities :: density_map(), ClippedDensities :: density_map()}. -type var_map() :: #{non_neg_integer() => map()}. -type hex_resolutions() :: [non_neg_integer()]. --type locations() :: [h3:h3_index()]. +-type locations() :: #{h3:h3_index() => [libp2p_crypto:pubkey_bin(),...]}. +-type h3_indices() :: [h3:h3_index()]. %%-------------------------------------------------------------------- %% Public functions @@ -31,14 +32,19 @@ densities(Ledger) -> ClippedDensities :: density_map() ) -> float(). scale(Location, TargetRes, UnclippedDensities, ClippedDensities) -> - lists:foldl( - fun(R, Acc) -> - Parent = h3:parent(Location, R), - Acc * (maps:get(Parent, ClippedDensities) / maps:get(Parent, UnclippedDensities)) - end, - 1.0, - lists:seq(TargetRes, 0, -1) - ). + case TargetRes >= h3:get_resolution(Location) of + true -> + maps:get(Location, ClippedDensities) / maps:get(Location, UnclippedDensities); + false -> + lists:foldl( + fun(R, Acc) -> + Parent = h3:parent(Location, R), + Acc * (maps:get(Parent, ClippedDensities) / maps:get(Parent, UnclippedDensities)) + end, + 1.0, + lists:seq(TargetRes, 0, -1) + ) + end. -spec var_map(Ledger :: blockchain_ledger_v1:ledger()) -> var_map(). var_map(Ledger) -> @@ -78,9 +84,7 @@ var_map(Ledger) -> -spec filtered_locations(Ledger :: blockchain_ledger_v1:ledger()) -> locations(). filtered_locations(Ledger) -> - AG = blockchain_ledger_v1:active_gateways(Ledger), - UnfilteredLocs = [blockchain_ledger_gateway_v2:location(G) || G <- maps:values(AG)], - lists:filter(fun(L) -> L /= undefined end, UnfilteredLocs). + blockchain_ledger_v1:get_h3dex(Ledger). -spec hex_resolutions(VarMap :: var_map()) -> hex_resolutions(). hex_resolutions(VarMap) -> @@ -93,9 +97,15 @@ densities(VarMap, Locations) -> [Head | Tail] = hex_resolutions(VarMap), %% find parent hexs to all hotspots at highest resolution in chain variables - ParentHexes = [h3:parent(Hex, Head) || Hex <- Locations], - - InitialDensities = init_densities(ParentHexes, #{}), + {ParentHexes, InitialDensities} = maps:fold(fun(Hex, GWs, {HAcc, MAcc}) -> + ParentHex = h3:parent(Hex, Head), + case maps:find(ParentHex, MAcc) of + error -> + {[ParentHex|HAcc], maps:put(ParentHex, length(GWs), MAcc)}; + {ok, OldCount} -> + {HAcc, maps:put(ParentHex, OldCount + length(GWs), MAcc)} + end + end, {[], #{}}, Locations), {UDensities, Densities} = build_densities( VarMap, @@ -106,49 +116,30 @@ densities(VarMap, Locations) -> {UDensities, Densities}. --spec init_densities(ParentHexes :: locations(), Init :: density_map()) -> density_map(). -init_densities(ParentHexes, Init) -> - lists:foldl( - fun(Hex, Acc) -> - maps:update_with( - Hex, - fun(V) -> V + 1 end, - 1, - Acc - ) - end, - Init, - ParentHexes - ). - --spec build_densities(var_map(), locations(), densities(), [non_neg_integer()]) -> densities(). +-spec build_densities(var_map(), h3_indices(), densities(), [non_neg_integer()]) -> densities(). build_densities(_VarMap, _ParentHexes, {UAcc, Acc}, []) -> {UAcc, Acc}; -build_densities(VarMap, ParentHexes, {UAcc, Acc}, [Res | Tail]) -> - ChildHexes = lists:usort(ParentHexes), - - ChildToParents = [{Hex, h3:parent(Hex, Res)} || Hex <- ChildHexes], +build_densities(VarMap, ChildHexes, {UAcc, Acc}, [Res | Tail]) -> + UD = unclipped_densities(ChildHexes, Res, Acc), + UM0 = maps:merge(UAcc, UD), + M0 = maps:merge(Acc, UD), - {UM0, M0} = unclipped_densities(ChildToParents, {UAcc, Acc}), + OccupiedHexesThisRes = maps:keys(UD), - OccupiedHexesThisRes = lists:usort([ThisParentHex || {_, ThisParentHex} <- ChildToParents]), + DensityTarget = maps:get(density_tgt, maps:get(Res, VarMap)), - {UM1, M1} = lists:foldl( - fun(ThisResHex, {UAcc3, Acc3}) -> - OccupiedCount = occupied_count(Res, VarMap, ThisResHex, M0), + M1 = lists:foldl( + fun(ThisResHex, Acc3) -> + OccupiedCount = occupied_count(DensityTarget, ThisResHex, UD), Limit = limit(Res, VarMap, OccupiedCount), - - { - maps:put(ThisResHex, maps:get(ThisResHex, M0), UAcc3), - maps:put(ThisResHex, min(Limit, maps:get(ThisResHex, M0)), Acc3) - } + maps:put(ThisResHex, min(Limit, maps:get(ThisResHex, M0)), Acc3) end, - {UM0, M0}, + M0, OccupiedHexesThisRes ), - build_densities(VarMap, OccupiedHexesThisRes, {UM1, M1}, Tail). + build_densities(VarMap, OccupiedHexesThisRes, {UM0, M1}, Tail). -spec limit( Res :: non_neg_integer(), @@ -163,19 +154,17 @@ limit(Res, VarMap, OccupiedCount) -> ). -spec occupied_count( - Res :: non_neg_integer(), - VarMap :: var_map(), + DensityTarget :: non_neg_integer(), ThisResHex :: h3:h3_index(), DensityMap :: density_map() ) -> non_neg_integer(). -occupied_count(Res, VarMap, ThisResHex, DensityMap) -> +occupied_count(DensityTarget, ThisResHex, DensityMap) -> H3Neighbors = h3:k_ring(ThisResHex, 1), lists:foldl( fun(Neighbor, Acc) -> ToAdd = case - maps:get(Neighbor, DensityMap, 0) >= - maps:get(density_tgt, maps:get(Res, VarMap)) + maps:get(Neighbor, DensityMap, 0) >= DensityTarget of false -> 0; true -> 1 @@ -186,24 +175,19 @@ occupied_count(Res, VarMap, ThisResHex, DensityMap) -> H3Neighbors ). --spec unclipped_densities(locations(), densities()) -> densities(). -unclipped_densities(ChildToParents, {UAcc, Acc}) -> +%-spec unclipped_densities(locations(), densities()) -> densities(). +unclipped_densities(ChildToParents, Res, Acc) -> lists:foldl( - fun({ChildHex, ThisParentHex}, {UAcc2, Acc2}) -> - {maps:update_with( - ThisParentHex, - fun(V) -> V + maps:get(ChildHex, Acc, 0) end, - maps:get(ChildHex, Acc, 0), - UAcc2 - ), + fun(ChildHex, Acc2) -> + ThisParentHex = h3:parent(ChildHex, Res), maps:update_with( - ThisParentHex, - fun(V) -> V + maps:get(ChildHex, Acc, 0) end, - maps:get(ChildHex, Acc, 0), - Acc2 - )} + ThisParentHex, + fun(V) -> V + maps:get(ChildHex, Acc, 0) end, + maps:get(ChildHex, Acc, 0), + Acc2 + ) end, - {UAcc, Acc}, + #{}, ChildToParents ). diff --git a/src/ledger/v1/blockchain_ledger_v1.erl b/src/ledger/v1/blockchain_ledger_v1.erl index 062ab20aef..46f774525d 100644 --- a/src/ledger/v1/blockchain_ledger_v1.erl +++ b/src/ledger/v1/blockchain_ledger_v1.erl @@ -120,7 +120,8 @@ clean_all_hexes/1, - set_h3dex/2, + bootstrap_h3dex/1, + get_h3dex/1, lookup_gateways_from_hex/2, add_gw_to_hex/3, remove_gw_from_hex/3, @@ -3108,31 +3109,57 @@ clean_all_hexes(Ledger) -> _ -> ok end. +-spec bootstrap_h3dex(ledger()) -> ok. +bootstrap_h3dex(Ledger) -> + AGwsCF = active_gateways_cf(Ledger), + H3Dex = cache_fold( + Ledger, + AGwsCF, + fun({GwAddr, Binary}, Acc) -> + Gw = blockchain_ledger_gateway_v2:deserialize(Binary), + case blockchain_ledger_gateway_v2:location(Gw) of + undefined -> + Acc; + Location -> + maps:update_with(Location, fun(V) -> [GwAddr | V] end, [GwAddr], Acc) + end + end, + #{}), + set_h3dex(H3Dex, Ledger). + -spec set_h3dex(h3dex(), ledger()) -> ok. set_h3dex(H3Dex, Ledger) -> H3CF = h3dex_cf(Ledger), _ = maps:map(fun(Loc, Gateways) -> - BinLoc = <>, + BinLoc = h3_to_key(Loc), BinGWs = term_to_binary(Gateways, [compressed]), cache_put(Ledger, H3CF, BinLoc, BinGWs) end, H3Dex), ok. +-spec get_h3dex(ledger()) -> h3dex(). +get_h3dex(Ledger) -> + H3CF = h3dex_cf(Ledger), + Res = cache_fold(Ledger, H3CF, + fun({Key, GWs}, Acc) -> + maps:put(key_to_h3(Key), binary_to_term(GWs), Acc) + end, #{}, []), + Res. + -spec lookup_gateways_from_hex(Hex :: non_neg_integer(), - Ledger :: ledger()) -> {ok, Results :: h3dex()}. + Ledger :: ledger()) -> Results :: h3dex(). %% @doc Given a hex find candidate gateways in the span to the next adjacent %% hex. N.B. May return an empty map. lookup_gateways_from_hex(Hex, Ledger) -> H3CF = h3dex_cf(Ledger), - Res = cache_fold(Ledger, H3CF, - fun({<>, GWs}, Acc) -> - maps:put(Loc, binary_to_term(GWs), Acc) - end, #{}, [ - {start, find_lower_bound_hex(Hex)}, - {iterate_upper_bound, parse_h3(Hex)} - ] - ), - {ok, Res}. + cache_fold(Ledger, H3CF, + fun({Key, GWs}, Acc) -> + maps:put(key_to_h3(Key), binary_to_term(GWs), Acc) + end, #{}, [ + {start, {seek, find_lower_bound_hex(Hex)}}, + {iterate_upper_bound, increment_bin(h3_to_key(Hex))} + ] + ). -spec find_lower_bound_hex(Hex :: non_neg_integer()) -> binary(). %% @doc Let's find the nearest set of k neighbors for this hex at the @@ -3140,12 +3167,27 @@ lookup_gateways_from_hex(Hex, Ledger) -> %% are actually packed binaries, we will destructure them to sort better %% lexically. find_lower_bound_hex(Hex) -> - parse_h3(hd(h3:children(Hex, 15))). + %% both reserved fields must be 0 and Mode must be 1 for this to be a h3 cell + <<0:1, 1:4/integer-unsigned-big, 0:3, Resolution:4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits/bitstring>> = <>, + ActualDigitCount = Resolution * 3, + %% pull out the actual digits used and dump the rest + <> = Digits, + Padding = 45 - ActualDigitCount, + %% store the resolution inverted (15 - 15) = 0 so it sorts earlier + %% pad the actual digits used with 0s on the end + <>. + +h3_to_key(H3) -> + %% both reserved fields must be 0 and Mode must be 1 for this to be a h3 cell + <<0:1/integer-unsigned-big, 1:4/integer-unsigned-big, 0:3/integer-unsigned-big, Resolution:4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits:45/integer-unsigned-big>> = <>, + %% store the resolution inverted (15 - Resolution) so it sorts later + <>. + +key_to_h3(Key) -> + <> = Key, + <> = <<0:1, 1:4/integer-unsigned-big, 0:3, (15 - InverseResolution):4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits:45/integer-unsigned-big>>, + H3. -parse_h3(H3) -> - <<_Reserved:1, _Mode:4, _Reserved2:3, - Resolution:4, BaseCell:7, Digits:45>> = <>, - <>. -spec add_gw_to_hex(Hex :: non_neg_integer(), GWAddr :: libp2p_crypto:pubkey_bin(), @@ -3153,7 +3195,7 @@ parse_h3(H3) -> %% @doc During an assert, this function will add a gateway address to a hex add_gw_to_hex(Hex, GWAddr, Ledger) -> H3CF = h3dex_cf(Ledger), - BinHex = <>, + BinHex = h3_to_key(Hex), case cache_get(Ledger, H3CF, BinHex, []) of not_found -> cache_put(Ledger, H3CF, BinHex, term_to_binary([GWAddr], [compressed])); @@ -3171,12 +3213,16 @@ add_gw_to_hex(Hex, GWAddr, Ledger) -> %% address from a hex remove_gw_from_hex(Hex, GWAddr, Ledger) -> H3CF = h3dex_cf(Ledger), - BinHex = <>, + BinHex = h3_to_key(Hex), case cache_get(Ledger, H3CF, BinHex, []) of not_found -> ok; {ok, BinGws} -> - NewGWs = lists:delete(GWAddr, binary_to_term(BinGws)), - cache_put(Ledger, H3CF, BinHex, term_to_binary(NewGWs, [compressed])); + case lists:delete(GWAddr, binary_to_term(BinGws)) of + [] -> + cache_delete(Ledger, H3CF, BinHex); + NewGWs -> + cache_put(Ledger, H3CF, BinHex, term_to_binary(NewGWs, [compressed])) + end; Error -> Error end. diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index e1fbe44cef..d0ff22105c 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -522,8 +522,12 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% check if there were any legitimate witnesses Witnesses = legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version), Challengee = blockchain_poc_path_element_v1:challengee(Elem), - {ok, ChallengeeGw} = blockchain_ledger_v1:find_gateway_info(Challengee, Ledger), - ChallengeeLoc = blockchain_ledger_gateway_v2:location(ChallengeeGw), + ChallengeeLoc = case blockchain_ledger_v1:find_gateway_info(Challengee, Ledger) of + {ok, ChallengeeGw} -> + blockchain_ledger_gateway_v2:location(ChallengeeGw); + _ -> + undefined + end, I = maps:get(Challengee, Acc0, 0), case blockchain_poc_path_element_v1:receipt(Elem) of undefined -> @@ -751,6 +755,7 @@ poc_witnesses_rewards(Transactions, lists:foldl( fun(WitnessRecord, Map) -> Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), + %% A witness must be on the ledger AND have a location, so this should be safe {ok, WitnessGw} = blockchain_ledger_v1:find_gateway_info(Witness, Ledger), WitnessLoc = blockchain_ledger_gateway_v2:location(WitnessGw), RxScale = blockchain_hex:scale(WitnessLoc, @@ -1018,6 +1023,16 @@ legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version) -> blockchain_poc_path_element_v1:witnesses(Elem) end. +maybe_calc_tx_scale(DensityTgtRes, + ChallengeeLoc, + UnclippedDensities, + ClippedDensities) -> + case {DensityTgtRes, ChallengeeLoc} of + {undefined, _} -> 1.0; + {_, undefined} -> 1.0; + {D, Loc} -> blockchain_hex:scale(Loc, D, UnclippedDensities, ClippedDensities) + end. + %%-------------------------------------------------------------------- %% @doc %% @end @@ -1210,7 +1225,7 @@ poc_challengees_rewards_1_test() -> {gateway, poc_challengees, <<"a">>} => 117, {gateway, poc_challengees, <<"b">>} => 233 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). poc_challengees_rewards_2_test() -> @@ -1269,7 +1284,7 @@ poc_challengees_rewards_2_test() -> {gateway, poc_challengees, <<"a">>} => 117, {gateway, poc_challengees, <<"b">>} => 233 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). poc_challengees_rewards_3_test() -> @@ -1337,7 +1352,7 @@ poc_challengees_rewards_3_test() -> %% c gets 2 shares {gateway, poc_challengees, <<"c">>} => 44 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). poc_witnesses_rewards_test() -> @@ -1393,7 +1408,7 @@ poc_witnesses_rewards_test() -> Rewards = #{{gateway,poc_witnesses,<<"a">>} => 25, {gateway,poc_witnesses,<<"b">>} => 25}, - ?assertEqual(Rewards, normalize_witness_rewards(poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}), EpochVars)), + ?assertEqual(Rewards, normalize_witness_rewards(poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}, #{}, #{}), EpochVars)), test_utils:cleanup_tmp_dir(BaseDir). old_poc_challengers_rewards_test() -> @@ -1449,7 +1464,7 @@ old_poc_challengees_rewards_version_1_test() -> {gateway, poc_challengees, <<"1">>} => 175, {gateway, poc_challengees, <<"2">>} => 175 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). old_poc_challengees_rewards_version_2_test() -> @@ -1495,7 +1510,7 @@ old_poc_challengees_rewards_version_2_test() -> {gateway, poc_challengees, <<"1">>} => 175, {gateway, poc_challengees, <<"2">>} => 175 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). old_poc_witnesses_rewards_test() -> @@ -1529,7 +1544,7 @@ old_poc_witnesses_rewards_test() -> {gateway, poc_witnesses, <<"1">>} => 25, {gateway, poc_witnesses, <<"2">>} => 25 }, - ?assertEqual(Rewards, normalize_witness_rewards(poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}), EpochVars)), + ?assertEqual(Rewards, normalize_witness_rewards(poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}, #{}, #{}), EpochVars)), test_utils:cleanup_tmp_dir(BaseDir). dc_rewards_test() -> @@ -1726,8 +1741,8 @@ dc_rewards_v3_spillover_test() -> %% compute the original rewards with no spillover ChallengerRewards = normalize_challenger_rewards(poc_challengers_rewards(AllTxns, Vars, #{}), Vars), - ChallengeeRewards = normalize_challengee_rewards(poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}), Vars), - WitnessRewards = normalize_witness_rewards(poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}), Vars), + ChallengeeRewards = normalize_challengee_rewards(poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars), + WitnessRewards = normalize_witness_rewards(poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars), ChallengersAward = trunc(maps:get(epoch_reward, Vars) * maps:get(poc_challengers_percent, Vars)), ?assertEqual(#{{gateway,poc_challengers,<<"X">>} => ChallengersAward}, ChallengerRewards), %% entire 15% allocation @@ -1743,8 +1758,8 @@ dc_rewards_v3_spillover_test() -> %% apply the DC remainder, if any to the other PoC categories pro rata SpilloverChallengerRewards = normalize_challenger_rewards(poc_challengers_rewards(AllTxns, Vars, #{}), NewVars), - SpilloverChallengeeRewards = normalize_challengee_rewards(poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}), NewVars), - SpilloverWitnessRewards = normalize_witness_rewards(poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}), NewVars), + SpilloverChallengeeRewards = normalize_challengee_rewards(poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}, #{}), NewVars), + SpilloverWitnessRewards = normalize_witness_rewards(poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}, #{}), NewVars), ChallengerSpilloverAward = erlang:round(DCRemainder * ((maps:get(poc_challengers_percent, Vars) / (maps:get(poc_challengees_percent, Vars) + maps:get(poc_witnesses_percent, Vars) + @@ -1792,15 +1807,3 @@ common_poc_vars() -> }. -endif. - -maybe_calc_tx_scale(DensityTgtRes, - ChallengeeLoc, - UnclippedDensities, - ClippedDensities) -> - case DensityTgtRes of - undefined -> 1.0; - D -> blockchain_hex:scale(ChallengeeLoc, - D, - UnclippedDensities, - ClippedDensities) - end. diff --git a/src/transactions/v1/blockchain_txn_vars_v1.erl b/src/transactions/v1/blockchain_txn_vars_v1.erl index 331dceba11..9fb91c5e81 100644 --- a/src/transactions/v1/blockchain_txn_vars_v1.erl +++ b/src/transactions/v1/blockchain_txn_vars_v1.erl @@ -1106,14 +1106,15 @@ validate_var(Var, Value) -> invalid_var(Var, Value). validate_hip17_vars(Value, Var) when is_binary(Value) -> - %% We expect the value of the variable to be in format: <<"int,int,int">> - case size(Value) of - 3 -> - case get_density_var(Value) of - {error, _}=E0 -> - lager:error("unable to get densit var, reason: ~p", [E0]), - throw({error, {invalid_density_var, Var, Value}}); - {ok, Res} -> + case get_density_var(Value) of + {error, _}=E0 -> + lager:error("unable to get densit var, reason: ~p", [E0]), + throw({error, {invalid_density_var, Var, Value}}); + {ok, Res} -> + case length(Res) == 3 of + false -> + throw({error, {invalid_size, Var, Value}}); + true -> [Siblings, DensityTgt, DensityMax] = Res, CheckSiblings = validate_int_min_max(Siblings, "siblings", 1, 1000), CheckDensityTgt = validate_int_min_max(DensityTgt, "density_tgt", 1, 200000), @@ -1138,9 +1139,7 @@ validate_hip17_vars(Value, Var) when is_binary(Value) -> end end end - end; - _ -> - throw({error, {invalid_size, Var, Value}}) + end end; validate_hip17_vars(Value, Var) -> throw({error, {invalid_format, Var, Value}}). diff --git a/test/blockchain_hex_SUITE.erl b/test/blockchain_hex_SUITE.erl index 801fdccdc3..a59bf4f7bf 100644 --- a/test/blockchain_hex_SUITE.erl +++ b/test/blockchain_hex_SUITE.erl @@ -17,7 +17,8 @@ non_zero_test/1, known_values_test/1, known_differences_test/1, - scale_test/1 + scale_test/1, + h3dex_test/1 ]). %% Values taken from python model @@ -90,7 +91,8 @@ all() -> non_zero_test, known_values_test, known_differences_test, - scale_test + scale_test, + h3dex_test ]. %%-------------------------------------------------------------------- @@ -100,6 +102,11 @@ init_per_suite(Config) -> LedgerURL = "https://blockchain-core.s3-us-west-1.amazonaws.com/ledger-586724.tar.gz", Ledger = blockchain_ct_utils:ledger(hip17_vars(), LedgerURL), + Ledger1 = blockchain_ledger_v1:new_context(Ledger), + blockchain:bootstrap_h3dex(Ledger1), + blockchain_ledger_v1:commit_context(Ledger1), + blockchain_ledger_v1:compact(Ledger), + %% Check that the pinned ledger is at the height we expect it to be {ok, 586724} = blockchain_ledger_v1:current_height(Ledger), @@ -117,8 +124,9 @@ init_per_suite(Config) -> ), ct:pal("density calculation time: ~pms", [Time / 1000]), + ct:pal("density took ~p", [Time]), %% Check that the time is less than 1000ms - ?assert(1000 =< Time), + %?assert(1000 =< Time), [ {ledger, Ledger}, @@ -162,10 +170,12 @@ non_zero_test(Config) -> known_values_test(Config) -> ClippedDensities = ?config(clipped, Config), + Ledger = ?config(ledger, Config), %% assert some known values calculated from the python model (thanks @para1!) true = lists:all( fun({Hex, Density}) -> + ct:pal("~p ~p", [Density, maps:size(blockchain_ledger_v1:lookup_gateways_from_hex(Hex, Ledger))]), Density == maps:get(Hex, ClippedDensities) end, ?KNOWN @@ -218,6 +228,27 @@ scale_test(Config) -> ok. +h3dex_test(Config) -> + Ledger = ?config(ledger, Config), + + %% A known hotspot hex at res=12, there's only one here + Hex = 631236347406370303, + HexPubkeyBin = <<0,161,86,254,148,82,27,153,2,52,158,118,1,178,133,150,238, + 135,228,40,114,253,149,194,89,170,68,170,122,230,130,196, + 139>>, + + Gateways = blockchain_ledger_v1:lookup_gateways_from_hex(Hex, Ledger), + + GotPubkeyBin = hd(maps:get(Hex, Gateways)), + + ?assertEqual(1, map_size(Gateways)), + ?assertEqual(GotPubkeyBin, HexPubkeyBin), + + ct:pal("Hex: ~p", [Hex]), + ct:pal("Gateways: ~p", [Gateways]), + + ok. + %%-------------------------------------------------------------------- %% CHAIN VARIABLES %%-------------------------------------------------------------------- From 9fd9ebccaecbcff1464fd3f2a808af6d707a7cd0 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Wed, 25 Nov 2020 15:21:38 -0800 Subject: [PATCH 04/18] Refactor hex API and add more tests - Case ladder for hip17 vars - Bootstrap hexes for tests - Fix init_per_testcase for comparison test - Enhance tests, fix some more reward txn bugs - Add export scale test - Add basic real-world perf test for rewards - Dont build the full density map - Add a temporary full cross verification test - Fix scaling - Refactor hex API, make dialyzer happy - Adhere to update hex API in tests --- src/blockchain_hex.erl | 281 ++++++----- src/ledger/v1/blockchain_ledger_v1.erl | 8 +- .../v1/blockchain_txn_rewards_v1.erl | 196 +++++--- test/blockchain_hex_SUITE.erl | 234 ++++++--- test/blockchain_reward_hip17_SUITE.erl | 459 ++++++++++++++++++ test/blockchain_reward_perf_SUITE.erl | 151 ++++++ 6 files changed, 1068 insertions(+), 261 deletions(-) create mode 100644 test/blockchain_reward_hip17_SUITE.erl create mode 100644 test/blockchain_reward_perf_SUITE.erl diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl index 4060cae020..96a1e30ac7 100644 --- a/src/blockchain_hex.erl +++ b/src/blockchain_hex.erl @@ -1,191 +1,213 @@ -module(blockchain_hex). --export([var_map/1, scale/4, densities/1]). +-export([var_map/1, scale/3]). + +-ifdef(TEST). +-export([densities/3]). +-endif. -include("blockchain_vars.hrl"). +-include_lib("common_test/include/ct.hrl"). + -type density_map() :: #{h3:h3_index() => pos_integer()}. -type densities() :: {UnclippedDensities :: density_map(), ClippedDensities :: density_map()}. --type var_map() :: #{non_neg_integer() => map()}. --type hex_resolutions() :: [non_neg_integer()]. --type locations() :: #{h3:h3_index() => [libp2p_crypto:pubkey_bin(),...]}. +-type var_map() :: #{0..12 => map()}. +-type locations() :: #{h3:h3_index() => [libp2p_crypto:pubkey_bin(), ...]}. -type h3_indices() :: [h3:h3_index()]. %%-------------------------------------------------------------------- %% Public functions %%-------------------------------------------------------------------- - --spec densities(Ledger :: blockchain_ledger_v1:ledger()) -> densities(). -densities(Ledger) -> - %% build a map of required chain variables, example: - %% #{0 => #{n => N, density_tgt => T, density_max =M}, ....} - VarMap = var_map(Ledger), - %% Filter out gateways with no location - Locations = filtered_locations(Ledger), - %% Calculate clipped and unclipped densities - densities(VarMap, Locations). - -spec scale( Location :: h3:h3_index(), - TargetRes :: non_neg_integer(), - UnclippedDensities :: density_map(), - ClippedDensities :: density_map() + VarMap :: var_map(), + Ledger :: blockchain_ledger_v1:ledger() ) -> float(). -scale(Location, TargetRes, UnclippedDensities, ClippedDensities) -> - case TargetRes >= h3:get_resolution(Location) of - true -> - maps:get(Location, ClippedDensities) / maps:get(Location, UnclippedDensities); - false -> - lists:foldl( - fun(R, Acc) -> - Parent = h3:parent(Location, R), - Acc * (maps:get(Parent, ClippedDensities) / maps:get(Parent, UnclippedDensities)) - end, - 1.0, - lists:seq(TargetRes, 0, -1) - ) - end. - --spec var_map(Ledger :: blockchain_ledger_v1:ledger()) -> var_map(). +scale(Location, VarMap, Ledger) -> + {UnclippedDensities, ClippedDensities} = densities(Location, VarMap, Ledger), + maps:get(Location, ClippedDensities) / maps:get(Location, UnclippedDensities). + +%% TODO: This ought to be stored in the ledger because it won't change much? ever? +%% after it's been computed. Seems dumb to calculate it every single time we pay +%% out rewards. +-spec var_map(Ledger :: blockchain_ledger_v1:ledger()) -> {error, any()} | {ok, var_map()}. var_map(Ledger) -> - [N0, Tgt0, Max0] = get_density_var(?hip17_res_0, Ledger), - [N1, Tgt1, Max1] = get_density_var(?hip17_res_1, Ledger), - [N2, Tgt2, Max2] = get_density_var(?hip17_res_2, Ledger), - [N3, Tgt3, Max3] = get_density_var(?hip17_res_3, Ledger), - [N4, Tgt4, Max4] = get_density_var(?hip17_res_4, Ledger), - [N5, Tgt5, Max5] = get_density_var(?hip17_res_5, Ledger), - [N6, Tgt6, Max6] = get_density_var(?hip17_res_6, Ledger), - [N7, Tgt7, Max7] = get_density_var(?hip17_res_7, Ledger), - [N8, Tgt8, Max8] = get_density_var(?hip17_res_8, Ledger), - [N9, Tgt9, Max9] = get_density_var(?hip17_res_9, Ledger), - [N10, Tgt10, Max10] = get_density_var(?hip17_res_10, Ledger), - [N11, Tgt11, Max11] = get_density_var(?hip17_res_11, Ledger), - [N12, Tgt12, Max12] = get_density_var(?hip17_res_12, Ledger), - - #{ - 0 => #{n => N0, density_tgt => Tgt0, density_max => Max0}, - 1 => #{n => N1, density_tgt => Tgt1, density_max => Max1}, - 2 => #{n => N2, density_tgt => Tgt2, density_max => Max2}, - 3 => #{n => N3, density_tgt => Tgt3, density_max => Max3}, - 4 => #{n => N4, density_tgt => Tgt4, density_max => Max4}, - 5 => #{n => N5, density_tgt => Tgt5, density_max => Max5}, - 6 => #{n => N6, density_tgt => Tgt6, density_max => Max6}, - 7 => #{n => N7, density_tgt => Tgt7, density_max => Max7}, - 8 => #{n => N8, density_tgt => Tgt8, density_max => Max8}, - 9 => #{n => N9, density_tgt => Tgt9, density_max => Max9}, - 10 => #{n => N10, density_tgt => Tgt10, density_max => Max10}, - 11 => #{n => N11, density_tgt => Tgt11, density_max => Max11}, - 12 => #{n => N12, density_tgt => Tgt12, density_max => Max12} - }. + ResolutionVars = [ + ?hip17_res_0, + ?hip17_res_1, + ?hip17_res_2, + ?hip17_res_3, + ?hip17_res_4, + ?hip17_res_5, + ?hip17_res_6, + ?hip17_res_7, + ?hip17_res_8, + ?hip17_res_9, + ?hip17_res_10, + ?hip17_res_11, + ?hip17_res_12 + ], + + {_I, Errors, M} = lists:foldl( + fun(A, {I, Errors, Acc}) -> + case get_density_var(A, Ledger) of + {error, _} = E -> + {I + 1, [{A, E} | Errors], Acc}; + {ok, [N, Tgt, Max]} -> + {I + 1, Errors, + maps:put( + I, + #{ + n => N, + tgt => Tgt, + max => Max + }, + Acc + )} + end + end, + {0, [], #{}}, + ResolutionVars + ), + + case Errors of + [] -> {ok, M}; + Errors -> {error, Errors} + end. %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +-spec densities( + H3Index :: h3:h3_index(), + VarMap :: var_map(), + Ledger :: blockchain_ledger_v1:ledger() +) -> densities(). +densities(H3Index, VarMap, Ledger) -> + Locations = blockchain_ledger_v1:lookup_gateways_from_hex(h3:k_ring(H3Index, 1), Ledger), + %% Calculate clipped and unclipped densities + densities(H3Index, VarMap, Locations, Ledger). --spec filtered_locations(Ledger :: blockchain_ledger_v1:ledger()) -> locations(). -filtered_locations(Ledger) -> - blockchain_ledger_v1:get_h3dex(Ledger). - --spec hex_resolutions(VarMap :: var_map()) -> hex_resolutions(). -hex_resolutions(VarMap) -> - %% [12, 11, ... 0] - lists:reverse(lists:sort(maps:keys(VarMap))). - --spec densities(VarMap :: var_map(), Locations :: locations()) -> densities(). -densities(VarMap, Locations) -> - %% Head = 12, Tail = [11, 10, ... 0] - [Head | Tail] = hex_resolutions(VarMap), - - %% find parent hexs to all hotspots at highest resolution in chain variables - {ParentHexes, InitialDensities} = maps:fold(fun(Hex, GWs, {HAcc, MAcc}) -> - ParentHex = h3:parent(Hex, Head), - case maps:find(ParentHex, MAcc) of - error -> - {[ParentHex|HAcc], maps:put(ParentHex, length(GWs), MAcc)}; - {ok, OldCount} -> - {HAcc, maps:put(ParentHex, OldCount + length(GWs), MAcc)} - end - end, {[], #{}}, Locations), - - {UDensities, Densities} = build_densities( - VarMap, - ParentHexes, - {InitialDensities, InitialDensities}, - Tail - ), - - {UDensities, Densities}. +-spec densities( + H3Root :: h3:h3_index(), + VarMap :: var_map(), + Locations :: locations(), + Ledger :: blockchain_ledger_v1:ledger() +) -> densities(). +densities(H3Root, VarMap, Locations, Ledger) -> + case maps:size(Locations) of + 0 -> + {#{}, #{}}; + _ -> + UpperBoundRes = lists:max([h3:get_resolution(H3) || H3 <- maps:keys(Locations)]), + LowerBoundRes = h3:get_resolution(H3Root), + ct:pal("UpperBoundRes: ~p, LowerBoundRes: ~p", [UpperBoundRes, LowerBoundRes]), + + [Head | Tail] = lists:seq(UpperBoundRes, LowerBoundRes, -1), + + %% find parent hexs to all hotspots at highest resolution in chain variables + {ParentHexes, InitialDensities} = + maps:fold( + fun(Hex, GWs, {HAcc, MAcc}) -> + ParentHex = h3:parent(Hex, Head), + case maps:find(ParentHex, MAcc) of + error -> + {[ParentHex | HAcc], maps:put(ParentHex, length(GWs), MAcc)}; + {ok, OldCount} -> + {HAcc, maps:put(ParentHex, OldCount + length(GWs), MAcc)} + end + end, + {[], #{}}, + Locations + ), + + build_densities( + H3Root, + Ledger, + VarMap, + ParentHexes, + {InitialDensities, InitialDensities}, + Tail + ) + end. --spec build_densities(var_map(), h3_indices(), densities(), [non_neg_integer()]) -> densities(). -build_densities(_VarMap, _ParentHexes, {UAcc, Acc}, []) -> +-spec build_densities( + h3:h3_index(), + blockchain_ledger_v1:ledger(), + var_map(), + h3_indices(), + densities(), + [non_neg_integer()] +) -> densities(). +build_densities(_H3Root, _Ledger, _VarMap, _ParentHexes, {UAcc, Acc}, []) -> {UAcc, Acc}; -build_densities(VarMap, ChildHexes, {UAcc, Acc}, [Res | Tail]) -> +build_densities(H3Root, Ledger, VarMap, ChildHexes, {UAcc, Acc}, [Res | Tail]) -> UD = unclipped_densities(ChildHexes, Res, Acc), UM0 = maps:merge(UAcc, UD), M0 = maps:merge(Acc, UD), OccupiedHexesThisRes = maps:keys(UD), - DensityTarget = maps:get(density_tgt, maps:get(Res, VarMap)), + DensityTarget = maps:get(tgt, maps:get(Res, VarMap)), M1 = lists:foldl( fun(ThisResHex, Acc3) -> OccupiedCount = occupied_count(DensityTarget, ThisResHex, UD), - Limit = limit(Res, VarMap, OccupiedCount), + + ct:pal("Limit: ~p, OccupiedCount: ~p", [Limit, OccupiedCount]), + maps:put(ThisResHex, min(Limit, maps:get(ThisResHex, M0)), Acc3) end, M0, OccupiedHexesThisRes ), - build_densities(VarMap, OccupiedHexesThisRes, {UM0, M1}, Tail). + build_densities(H3Root, Ledger, VarMap, OccupiedHexesThisRes, {UM0, M1}, Tail). -spec limit( - Res :: non_neg_integer(), + Res :: 0..12, VarMap :: var_map(), OccupiedCount :: non_neg_integer() ) -> non_neg_integer(). limit(Res, VarMap, OccupiedCount) -> min( - maps:get(density_max, maps:get(Res, VarMap)), - maps:get(density_tgt, maps:get(Res, VarMap)) * + maps:get(max, maps:get(Res, VarMap)), + maps:get(tgt, maps:get(Res, VarMap)) * max((OccupiedCount - maps:get(n, maps:get(Res, VarMap))), 1) ). -spec occupied_count( - DensityTarget :: non_neg_integer(), + DensityTarget :: 0..12, ThisResHex :: h3:h3_index(), DensityMap :: density_map() ) -> non_neg_integer(). occupied_count(DensityTarget, ThisResHex, DensityMap) -> H3Neighbors = h3:k_ring(ThisResHex, 1), + lists:foldl( fun(Neighbor, Acc) -> - ToAdd = - case - maps:get(Neighbor, DensityMap, 0) >= DensityTarget - of - false -> 0; - true -> 1 - end, - Acc + ToAdd + case maps:get(Neighbor, DensityMap, 0) >= DensityTarget of + false -> Acc; + true -> Acc + 1 + end end, 0, H3Neighbors ). -%-spec unclipped_densities(locations(), densities()) -> densities(). +-spec unclipped_densities(h3_indices(), 0..12, density_map()) -> density_map(). unclipped_densities(ChildToParents, Res, Acc) -> lists:foldl( fun(ChildHex, Acc2) -> - ThisParentHex = h3:parent(ChildHex, Res), - maps:update_with( - ThisParentHex, - fun(V) -> V + maps:get(ChildHex, Acc, 0) end, - maps:get(ChildHex, Acc, 0), - Acc2 - ) + ThisParentHex = h3:parent(ChildHex, Res), + maps:update_with( + ThisParentHex, + fun(V) -> V + maps:get(ChildHex, Acc, 0) end, + maps:get(ChildHex, Acc, 0), + Acc2 + ) end, #{}, ChildToParents @@ -194,8 +216,15 @@ unclipped_densities(ChildToParents, Res, Acc) -> -spec get_density_var( Var :: atom(), Ledger :: blockchain_ledger_v1:ledger() -) -> [pos_integer()]. +) -> {error, any()} | {ok, [pos_integer()]}. get_density_var(Var, Ledger) -> - {ok, Bin} = blockchain:config(Var, Ledger), - [N, Tgt, Max] = [list_to_integer(I) || I <- string:tokens(binary:bin_to_list(Bin), ",")], - [N, Tgt, Max]. + case blockchain:config(Var, Ledger) of + {error, _} = E -> + E; + {ok, Bin} -> + [N, Tgt, Max] = [ + list_to_integer(I) + || I <- string:tokens(binary:bin_to_list(Bin), ",") + ], + {ok, [N, Tgt, Max]} + end. diff --git a/src/ledger/v1/blockchain_ledger_v1.erl b/src/ledger/v1/blockchain_ledger_v1.erl index 46f774525d..a438604955 100644 --- a/src/ledger/v1/blockchain_ledger_v1.erl +++ b/src/ledger/v1/blockchain_ledger_v1.erl @@ -3146,11 +3146,15 @@ get_h3dex(Ledger) -> end, #{}, []), Res. --spec lookup_gateways_from_hex(Hex :: non_neg_integer(), +-spec lookup_gateways_from_hex(Hex :: [non_neg_integer()] | non_neg_integer(), Ledger :: ledger()) -> Results :: h3dex(). %% @doc Given a hex find candidate gateways in the span to the next adjacent %% hex. N.B. May return an empty map. -lookup_gateways_from_hex(Hex, Ledger) -> +lookup_gateways_from_hex(Hexes, Ledger) when is_list(Hexes) -> + lists:foldl(fun(Hex, Acc) -> + maps:merge(Acc, lookup_gateways_from_hex(Hex, Ledger)) + end, #{}, Hexes); +lookup_gateways_from_hex(Hex, Ledger) when is_integer(Hex) -> H3CF = h3dex_cf(Ledger), cache_fold(Ledger, H3CF, fun({Key, GWs}, Acc) -> diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index d0ff22105c..19083b994a 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -250,12 +250,35 @@ get_rewards_for_epoch(Current, End, Chain, Vars, Ledger, ChallengerRewards, Chal true -> {error, already_existing_rewards}; false -> - {UnclippedDensities, ClippedDensities} = blockchain_hex:densities(Ledger), - get_rewards_for_epoch(Current+1, End, Chain, Vars, Ledger, - poc_challengers_rewards(Transactions, Vars, ChallengerRewards), - poc_challengees_rewards(Transactions, Vars, Chain, Ledger, ChallengeeRewards, UnclippedDensities, ClippedDensities), - poc_witnesses_rewards(Transactions, Vars, Chain, Ledger, WitnessRewards, UnclippedDensities, ClippedDensities), - dc_rewards(Transactions, End, Vars, Ledger, DCRewards)) + case blockchain_hex:var_map(Ledger) of + {error, _} -> + %% do the old thing + get_rewards_for_epoch(Current+1, End, Chain, Vars, Ledger, + poc_challengers_rewards(Transactions, Vars, + ChallengerRewards), + poc_challengees_rewards(Transactions, Vars, + Chain, Ledger, + ChallengeeRewards, #{}), + poc_witnesses_rewards(Transactions, Vars, + Chain, Ledger, + WitnessRewards, #{}), + dc_rewards(Transactions, End, Vars, + Ledger, DCRewards)); + {ok, VarMap} -> + %% do the new thing + get_rewards_for_epoch(Current+1, End, Chain, Vars, Ledger, + poc_challengers_rewards(Transactions, Vars, + ChallengerRewards), + poc_challengees_rewards(Transactions, Vars, + Chain, Ledger, + ChallengeeRewards, + VarMap), + poc_witnesses_rewards(Transactions, Vars, + Chain, Ledger, + WitnessRewards, VarMap), + dc_rewards(Transactions, End, Vars, + Ledger, DCRewards)) + end end end. @@ -464,15 +487,14 @@ normalize_challenger_rewards(ChallengerRewards, #{epoch_reward := EpochReward, Chain :: blockchain:blockchain(), Ledger :: blockchain_ledger_v1:ledger(), ExistingRewards :: map(), - UnclippedDensities :: blockchain_hex:density_map(), - ClippedDensities :: blockchain_hex:density_map()) -> #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. + VarMap :: blockchain_hex:var_map()) -> + #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. poc_challengees_rewards(Transactions, Vars, Chain, Ledger, ExistingRewards, - UnclippedDensities, - ClippedDensities) -> + VarMap) -> lists:foldl( fun(Txn, Acc0) -> case blockchain_txn:type(Txn) == blockchain_txn_poc_receipts_v1 of @@ -480,7 +502,7 @@ poc_challengees_rewards(Transactions, Acc0; true -> Path = blockchain_txn_poc_receipts_v1:path(Txn), - poc_challengees_rewards_(Vars, Path, Path, Txn, Chain, Ledger, true, UnclippedDensities, ClippedDensities, Acc0) + poc_challengees_rewards_(Vars, Path, Path, Txn, Chain, Ledger, true, VarMap, Acc0) end end, ExistingRewards, @@ -504,7 +526,7 @@ normalize_challengee_rewards(ChallengeeRewards, #{epoch_reward := EpochReward, ChallengeeRewards ). -poc_challengees_rewards_(_Vars, [], _StaticPath, _Txn, _Chain, _Ledger, _, _, _, Acc) -> +poc_challengees_rewards_(_Vars, [], _StaticPath, _Txn, _Chain, _Ledger, _, _, Acc) -> Acc; poc_challengees_rewards_(#{poc_version := Version}=Vars, [Elem|Path], @@ -513,8 +535,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, Chain, Ledger, IsFirst, - UnclippedDensities, - ClippedDensities, + VarMap, Acc0) when Version >= 2 -> WitnessRedundancy = maps:get(witness_redundancy, Vars, undefined), DecayRate = maps:get(poc_reward_decay_rate, Vars, undefined), @@ -544,10 +565,11 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+1, Acc0); {ok, ToAdd} -> - TxScale = maybe_calc_tx_scale(DensityTgtRes, + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, ChallengeeLoc, - UnclippedDensities, - ClippedDensities), + VarMap, + Ledger), maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; true when is_integer(Version), Version > 4, IsFirst == false -> @@ -561,16 +583,17 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+2, Acc0); {ok, ToAdd} -> - TxScale = maybe_calc_tx_scale(DensityTgtRes, + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, ChallengeeLoc, - UnclippedDensities, - ClippedDensities), + VarMap, + Ledger), maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; _ -> Acc0 end, - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, UnclippedDensities, ClippedDensities, Acc1); + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1); Receipt -> case blockchain_poc_receipt_v1:origin(Receipt) of radio -> @@ -587,10 +610,11 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+3, Acc0); {ok, ToAdd} -> - TxScale = maybe_calc_tx_scale(DensityTgtRes, + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, ChallengeeLoc, - UnclippedDensities, - ClippedDensities), + VarMap, + Ledger), maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; false when is_integer(Version), Version > 4 -> @@ -600,10 +624,11 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+2, Acc0); {ok, ToAdd} -> - TxScale = maybe_calc_tx_scale(DensityTgtRes, + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, ChallengeeLoc, - UnclippedDensities, - ClippedDensities), + VarMap, + Ledger), maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; _ -> @@ -612,14 +637,15 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+1, Acc0); {ok, ToAdd} -> - TxScale = maybe_calc_tx_scale(DensityTgtRes, + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, ChallengeeLoc, - UnclippedDensities, - ClippedDensities), + VarMap, + Ledger), maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end end, - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, UnclippedDensities, ClippedDensities, Acc1); + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1); p2p -> %% if there are legitimate witnesses or the path continues %% the challengee did their job @@ -637,10 +663,11 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+2, Acc0); {ok, ToAdd} -> - TxScale = maybe_calc_tx_scale(DensityTgtRes, + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, ChallengeeLoc, - UnclippedDensities, - ClippedDensities), + VarMap, + Ledger), maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; true -> @@ -649,25 +676,26 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+1, Acc0); {ok, ToAdd} -> - TxScale = maybe_calc_tx_scale(DensityTgtRes, + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, ChallengeeLoc, - UnclippedDensities, - ClippedDensities), + VarMap, + Ledger), maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end end, - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, UnclippedDensities, ClippedDensities, Acc1) + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1) end end; -poc_challengees_rewards_(Vars, [Elem|Path], StaticPath, Txn, Chain, Ledger, _IsFirst, UnclippedDensities, ClippedDensities, Acc0) -> +poc_challengees_rewards_(Vars, [Elem|Path], StaticPath, Txn, Chain, Ledger, _IsFirst, VarMap, Acc0) -> case blockchain_poc_path_element_v1:receipt(Elem) of undefined -> - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, UnclippedDensities, ClippedDensities, Acc0); + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc0); _Receipt -> Challengee = blockchain_poc_path_element_v1:challengee(Elem), I = maps:get(Challengee, Acc0, 0), Acc1 = maps:put(Challengee, I+1, Acc0), - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, UnclippedDensities, ClippedDensities, Acc1) + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1) end. -spec poc_challengee_reward_unit(WitnessRedundancy :: undefined | pos_integer(), @@ -681,23 +709,26 @@ poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) -> W = length(Witnesses), Unit = poc_reward_tx_unit(R, W, N), lager:info("poc_challengee_reward_unit: ~p", [Unit]), - {ok, Unit} + {ok, normalize_reward_unit(Unit)} end. +-spec normalize_reward_unit(Unit :: float()) -> float(). +normalize_reward_unit(Unit) when Unit > 1.0 -> 1.0; +normalize_reward_unit(Unit) -> Unit. + -spec poc_witnesses_rewards(Transactions :: blockchain_txn:txns(), Vars :: map(), Chain :: blockchain:blockchain(), Ledger :: blockchain_ledger_v1:ledger(), WitnessRewards :: map(), - UnclippedDensities :: blockchain_hex:density_map(), - ClippedDensities :: blockchain_hex:density_map()) -> #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. + VarMap :: blockchain_hex:var_map()) -> + #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. poc_witnesses_rewards(Transactions, #{poc_version := POCVersion}=Vars, Chain, Ledger, WitnessRewards, - UnclippedDensities, - ClippedDensities) -> + VarMap) -> WitnessRedundancy = maps:get(witness_redundancy, Vars, undefined), DecayRate = maps:get(poc_reward_decay_rate, Vars, undefined), DensityTgtRes = maps:get(density_tgt_res, Vars, undefined), @@ -733,8 +764,8 @@ poc_witnesses_rewards(Transactions, {_, undefined} -> 1; {N, R} -> W = length(ValidWitnesses), - U = poc_reward_rx_unit(R, W, N), - lager:info("poc_reward_rx_unit: ~p", [U]), + U = poc_witness_reward_unit(R, W, N), + lager:info("poc_witness_reward_unit: ~p", [U]), U end, @@ -750,7 +781,7 @@ poc_witnesses_rewards(Transactions, Acc1, ValidWitnesses ); - D -> + _D -> %% new (HIP17) lists:foldl( fun(WitnessRecord, Map) -> @@ -759,9 +790,10 @@ poc_witnesses_rewards(Transactions, {ok, WitnessGw} = blockchain_ledger_v1:find_gateway_info(Witness, Ledger), WitnessLoc = blockchain_ledger_gateway_v2:location(WitnessGw), RxScale = blockchain_hex:scale(WitnessLoc, - D, - UnclippedDensities, - ClippedDensities), + VarMap, + Ledger), + lager:info("WitnessGw: ~p, RxScale: ~p", [blockchain_utils:addr2name(Witness), + RxScale]), I = maps:get(Witness, Map, 0), maps:put(Witness, I+(ToAdd*RxScale), Map) end, @@ -993,13 +1025,13 @@ poc_reward_tx_unit(R, W, N) -> lager:info("nonorm, R: ~p, W: ~p, N: ~p, poc_reward_tx_unit: ~p", [R, W, N, NoNorm]), blockchain_utils:normalize_float(NoNorm). --spec poc_reward_rx_unit(R :: float(), - W :: pos_integer(), - N :: pos_integer()) -> float(). -poc_reward_rx_unit(_R, W, N) when W =< N -> - 1; -poc_reward_rx_unit(R, W, N) -> - blockchain_utils:normalize_float((N - (1 - math:pow(R, (W - N))))/W). +-spec poc_witness_reward_unit(R :: float(), + W :: pos_integer(), + N :: pos_integer()) -> float(). +poc_witness_reward_unit(_R, W, N) when W =< N -> + 1.0; +poc_witness_reward_unit(R, W, N) -> + normalize_reward_unit(blockchain_utils:normalize_float((N - (1 - math:pow(R, (W - N))))/W)). legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version) -> case Version of @@ -1023,14 +1055,19 @@ legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version) -> blockchain_poc_path_element_v1:witnesses(Elem) end. -maybe_calc_tx_scale(DensityTgtRes, +maybe_calc_tx_scale(Challengee, + DensityTgtRes, ChallengeeLoc, - UnclippedDensities, - ClippedDensities) -> + VarMap, + Ledger) -> case {DensityTgtRes, ChallengeeLoc} of {undefined, _} -> 1.0; {_, undefined} -> 1.0; - {D, Loc} -> blockchain_hex:scale(Loc, D, UnclippedDensities, ClippedDensities) + {_D, Loc} -> + TxScale = blockchain_hex:scale(Loc, VarMap, Ledger), + lager:info("Challengee: ~p, RxScale: ~p", [blockchain_utils:addr2name(Challengee), + TxScale]), + TxScale end. %%-------------------------------------------------------------------- @@ -1225,7 +1262,8 @@ poc_challengees_rewards_1_test() -> {gateway, poc_challengees, <<"a">>} => 117, {gateway, poc_challengees, <<"b">>} => 233 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards( + poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). poc_challengees_rewards_2_test() -> @@ -1284,7 +1322,8 @@ poc_challengees_rewards_2_test() -> {gateway, poc_challengees, <<"a">>} => 117, {gateway, poc_challengees, <<"b">>} => 233 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards( + poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, {}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). poc_challengees_rewards_3_test() -> @@ -1352,7 +1391,8 @@ poc_challengees_rewards_3_test() -> %% c gets 2 shares {gateway, poc_challengees, <<"c">>} => 44 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards( + poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, {}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). poc_witnesses_rewards_test() -> @@ -1408,7 +1448,8 @@ poc_witnesses_rewards_test() -> Rewards = #{{gateway,poc_witnesses,<<"a">>} => 25, {gateway,poc_witnesses,<<"b">>} => 25}, - ?assertEqual(Rewards, normalize_witness_rewards(poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}, #{}, #{}), EpochVars)), + ?assertEqual(Rewards, normalize_witness_rewards( + poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}, #{}), EpochVars)), test_utils:cleanup_tmp_dir(BaseDir). old_poc_challengers_rewards_test() -> @@ -1464,7 +1505,8 @@ old_poc_challengees_rewards_version_1_test() -> {gateway, poc_challengees, <<"1">>} => 175, {gateway, poc_challengees, <<"2">>} => 175 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards( + poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). old_poc_challengees_rewards_version_2_test() -> @@ -1510,7 +1552,8 @@ old_poc_challengees_rewards_version_2_test() -> {gateway, poc_challengees, <<"1">>} => 175, {gateway, poc_challengees, <<"2">>} => 175 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards( + poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). old_poc_witnesses_rewards_test() -> @@ -1544,7 +1587,8 @@ old_poc_witnesses_rewards_test() -> {gateway, poc_witnesses, <<"1">>} => 25, {gateway, poc_witnesses, <<"2">>} => 25 }, - ?assertEqual(Rewards, normalize_witness_rewards(poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}, #{}, #{}), EpochVars)), + ?assertEqual(Rewards, normalize_witness_rewards( + poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}, #{}), EpochVars)), test_utils:cleanup_tmp_dir(BaseDir). dc_rewards_test() -> @@ -1741,8 +1785,10 @@ dc_rewards_v3_spillover_test() -> %% compute the original rewards with no spillover ChallengerRewards = normalize_challenger_rewards(poc_challengers_rewards(AllTxns, Vars, #{}), Vars), - ChallengeeRewards = normalize_challengee_rewards(poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars), - WitnessRewards = normalize_witness_rewards(poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}, #{}), Vars), + ChallengeeRewards = normalize_challengee_rewards( + poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}), Vars), + WitnessRewards = normalize_witness_rewards( + poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}), Vars), ChallengersAward = trunc(maps:get(epoch_reward, Vars) * maps:get(poc_challengers_percent, Vars)), ?assertEqual(#{{gateway,poc_challengers,<<"X">>} => ChallengersAward}, ChallengerRewards), %% entire 15% allocation @@ -1758,8 +1804,10 @@ dc_rewards_v3_spillover_test() -> %% apply the DC remainder, if any to the other PoC categories pro rata SpilloverChallengerRewards = normalize_challenger_rewards(poc_challengers_rewards(AllTxns, Vars, #{}), NewVars), - SpilloverChallengeeRewards = normalize_challengee_rewards(poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}, #{}), NewVars), - SpilloverWitnessRewards = normalize_witness_rewards(poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}, #{}), NewVars), + SpilloverChallengeeRewards = normalize_challengee_rewards( + poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}), NewVars), + SpilloverWitnessRewards = normalize_witness_rewards( + poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}), NewVars), ChallengerSpilloverAward = erlang:round(DCRemainder * ((maps:get(poc_challengers_percent, Vars) / (maps:get(poc_challengees_percent, Vars) + maps:get(poc_witnesses_percent, Vars) + diff --git a/test/blockchain_hex_SUITE.erl b/test/blockchain_hex_SUITE.erl index a59bf4f7bf..63aab00786 100644 --- a/test/blockchain_hex_SUITE.erl +++ b/test/blockchain_hex_SUITE.erl @@ -14,11 +14,12 @@ ]). -export([ - non_zero_test/1, + full_known_values_test/1, known_values_test/1, known_differences_test/1, scale_test/1, - h3dex_test/1 + h3dex_test/1, + export_scale_test/1 ]). %% Values taken from python model @@ -88,11 +89,11 @@ all() -> [ - non_zero_test, known_values_test, known_differences_test, scale_test, h3dex_test + %% export_scale_test ]. %%-------------------------------------------------------------------- @@ -111,27 +112,12 @@ init_per_suite(Config) -> {ok, 586724} = blockchain_ledger_v1:current_height(Ledger), %% Check that the vars are correct, one is enough... - VarMap = blockchain_hex:var_map(Ledger), - Res4 = maps:get(4, VarMap), - 1 = maps:get(n, Res4), - 250 = maps:get(density_tgt, Res4), - 800 = maps:get(density_max, Res4), - - {Time, {UnclippedDensities, ClippedDensities}} = timer:tc( - fun() -> - blockchain_hex:densities(Ledger) - end - ), - ct:pal("density calculation time: ~pms", [Time / 1000]), - - ct:pal("density took ~p", [Time]), - %% Check that the time is less than 1000ms - %?assert(1000 =< Time), + {ok, VarMap} = blockchain_hex:var_map(Ledger), + ct:pal("var_map: ~p", [VarMap]), + #{ n := 1, tgt := 250, max := 800} = maps:get(4, VarMap), [ {ledger, Ledger}, - {clipped, ClippedDensities}, - {unclipped, UnclippedDensities}, {var_map, VarMap} | Config ]. @@ -160,23 +146,73 @@ end_per_suite(_Config) -> %% TEST CASES %%-------------------------------------------------------------------- -non_zero_test(Config) -> - ClippedDensities = ?config(clipped, Config), +%% XXX: Remove this test when done cross-checking with python +full_known_values_test(Config) -> + Ledger = ?config(ledger, Config), + + {ok, [List]} = file:consult("/tmp/tracker.erl"), + + %% assert some known values calculated from the python model (thanks @para1!) + true = lists:all( + fun({Hex, Res, UnclippedValue, _Limit, ClippedValue}) -> + case h3:get_resolution(Hex) of + 0 -> + true; + _ -> + {ok, VarMap} = blockchain_hex:var_map(Ledger), + + {UnclippedDensities, ClippedDensities} = blockchain_hex:densities( + Hex, + VarMap, + Ledger + ), + + Dex = blockchain_ledger_v1:lookup_gateways_from_hex(Hex, Ledger), + Spots = [blockchain_utils:addr2name(I) || I <- lists:flatten(maps:values(Dex))], + NumSpots = length(Spots), + + ct:pal( + "Hex: ~p, Res: ~p, PythonUnclippedDensity: ~p, CalculatedUnclippedDensity: ~p, PythonClippedDensity: ~p, CalculatedClippedDensity: ~p, NumSpots: ~p, Spots: ~p", + [ + Hex, + Res, + UnclippedValue, + maps:get(Hex, UnclippedDensities), + ClippedValue, + maps:get(Hex, ClippedDensities), + NumSpots, + Spots + ] + ), + + UnclippedValue == maps:get(Hex, UnclippedDensities) andalso ClippedValue == maps:get(Hex, ClippedDensities) + end + end, + List + ), - %% assert that there are no 0 values for density - %% we count a hotspot at res 12 (max resolution) as 1 - true = lists:all(fun(V) -> V /= 0 end, maps:values(ClippedDensities)), ok. known_values_test(Config) -> - ClippedDensities = ?config(clipped, Config), Ledger = ?config(ledger, Config), %% assert some known values calculated from the python model (thanks @para1!) true = lists:all( fun({Hex, Density}) -> - ct:pal("~p ~p", [Density, maps:size(blockchain_ledger_v1:lookup_gateways_from_hex(Hex, Ledger))]), - Density == maps:get(Hex, ClippedDensities) + case h3:get_resolution(Hex) of + 0 -> + true; + _ -> + {ok, VarMap} = blockchain_hex:var_map(Ledger), + {_, ClippedDensities} = blockchain_hex:densities(Hex, VarMap, Ledger), + + ct:pal("~p ~p", [ + Density, + maps:get(Hex, ClippedDensities) + ]), + + Density == maps:get(Hex, ClippedDensities) + end end, ?KNOWN ), @@ -184,29 +220,19 @@ known_values_test(Config) -> ok. known_differences_test(Config) -> - UnclippedDensities = ?config(unclipped, Config), - ClippedDensities = ?config(clipped, Config), - - Differences = lists:foldl( - fun({Hex, ClippedDensity}, Acc) -> - UnclippedDensity = maps:get(Hex, UnclippedDensities), - case ClippedDensity /= UnclippedDensity of - false -> - Acc; - true -> - maps:put(Hex, {ClippedDensity, UnclippedDensity}, Acc) - end - end, - #{}, - maps:to_list(ClippedDensities) - ), - - ?assertEqual(?KNOWN_CLIP_VS_UNCLIPPED, map_size(Differences)), + Ledger = ?config(ledger, Config), true = lists:all( fun({Hex, {Clipped, Unclipped}}) -> - Unclipped == maps:get(Hex, UnclippedDensities) andalso - Clipped == maps:get(Hex, ClippedDensities) + {ok, VarMap} = blockchain_hex:var_map(Ledger), + {UnclippedDensities, ClippedDensities} = blockchain_hex:densities(Hex, VarMap, Ledger), + GotUnclipped = maps:get(Hex, UnclippedDensities), + GotClipped = maps:get(Hex, ClippedDensities), + ct:pal( + "Hex: ~p, Clipped: ~p, GotClipped: ~p, Unclipped: ~p, GotUnclipped: ~p", + [Hex, Clipped, GotClipped, Unclipped, GotUnclipped] + ), + Unclipped == GotUnclipped andalso Clipped == GotClipped end, maps:to_list(?KNOWN_DIFFERENCES) ), @@ -214,15 +240,11 @@ known_differences_test(Config) -> ok. scale_test(Config) -> - UnclippedDensities = ?config(unclipped, Config), - ClippedDensities = ?config(clipped, Config), - + Ledger = ?config(ledger, Config), Another = h3:from_string("8c2836152804dff"), - ok = lists:foreach(fun(I) -> - Scale = blockchain_hex:scale(Another, I, UnclippedDensities, ClippedDensities), - ct:pal("Res: ~p, Scale: ~p", [I, Scale]) - end, lists:seq(12, 0, -1)), + Scale = blockchain_hex:scale(Another, Ledger), + ct:pal("Res: ~p, Scale: ~p", [h3:get_resolution(Another), Scale]), %% TODO: Assert checks from the python model @@ -233,9 +255,9 @@ h3dex_test(Config) -> %% A known hotspot hex at res=12, there's only one here Hex = 631236347406370303, - HexPubkeyBin = <<0,161,86,254,148,82,27,153,2,52,158,118,1,178,133,150,238, - 135,228,40,114,253,149,194,89,170,68,170,122,230,130,196, - 139>>, + HexPubkeyBin = + <<0, 161, 86, 254, 148, 82, 27, 153, 2, 52, 158, 118, 1, 178, 133, 150, 238, 135, 228, 40, + 114, 253, 149, 194, 89, 170, 68, 170, 122, 230, 130, 196, 139>>, Gateways = blockchain_ledger_v1:lookup_gateways_from_hex(Hex, Ledger), @@ -249,6 +271,23 @@ h3dex_test(Config) -> ok. +export_scale_test(Config) -> + Ledger = ?config(ledger, Config), + + %% A list of possible density target resolution we'd output the scales at + DensityTargetResolutions = lists:seq(3, 10), + + %% Only do this for gateways with known locations + GatewaysWithLocs = gateways_with_locs(Ledger), + + ok = export_scale_data( + Ledger, + DensityTargetResolutions, + GatewaysWithLocs + ), + + ok. + %%-------------------------------------------------------------------- %% CHAIN VARIABLES %%-------------------------------------------------------------------- @@ -270,3 +309,80 @@ hip17_vars() -> hip17_res_12 => <<"2,100000,100000">>, density_tgt_res => 8 }. + +%%-------------------------------------------------------------------- +%% INTERNAL FUNCTIONS +%%-------------------------------------------------------------------- + +gateways_with_locs(Ledger) -> + AG = blockchain_ledger_v1:active_gateways(Ledger), + + maps:fold( + fun(Addr, GW, Acc) -> + case blockchain_ledger_gateway_v2:location(GW) of + undefined -> Acc; + Loc -> [{blockchain_utils:addr2name(Addr), Loc} | Acc] + end + end, + [], + AG + ). + +export_scale_data(Ledger, DensityTargetResolutions, GatewaysWithLocs) -> + %% Calculate scale at each density target res for eventual comparison + lists:foreach( + fun(TargetRes) -> + %% Export scale data for every single gateway to a gps file + Scales = lists:foldl( + fun({GwName, Loc}, Acc) -> + Scale = blockchain_hex:scale( + Loc, + Ledger + ), + [{GwName, Loc, Scale} | Acc] + end, + [], + GatewaysWithLocs + ), + + Fname = "/tmp/scale_" ++ integer_to_list(TargetRes), + ok = export_gps_file(Fname, Scales) + end, + DensityTargetResolutions + ). + +export_gps_file(Fname, Scales) -> + Header = ["name,latitude,longitude,color,desc"], + + Data = lists:foldl( + fun({Name, H3, ScaleVal}, Acc) -> + {Lat, Long} = h3:to_geo(H3), + ToAppend = + Name ++ + "," ++ + io_lib:format("~.20f", [Lat]) ++ + "," ++ + io_lib:format("~.20f", [Long]) ++ + "," ++ + color(ScaleVal) ++ + "," ++ + io_lib:format("scale: ~p", [ScaleVal]), + [ToAppend | Acc] + end, + [], + Scales + ), + + TotalData = Header ++ Data, + + LineSep = io_lib:nl(), + Print = [string:join(TotalData, LineSep), LineSep], + file:write_file(Fname, Print), + ok. + +color(1.0) -> "green"; +color(V) when V =< 0.1 -> "red"; +color(V) when V =< 0.3 -> "orange"; +color(V) when V =< 0.5 -> "yellow"; +color(V) when V =< 0.8 -> "cyan"; +color(_) -> "blue". diff --git a/test/blockchain_reward_hip17_SUITE.erl b/test/blockchain_reward_hip17_SUITE.erl new file mode 100644 index 0000000000..8d275ebd9c --- /dev/null +++ b/test/blockchain_reward_hip17_SUITE.erl @@ -0,0 +1,459 @@ +-module(blockchain_reward_hip17_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-include("blockchain_vars.hrl"). + +-export([ + all/0, + init_per_suite/1, + end_per_suite/1, + init_per_testcase/2, + end_per_testcase/2 +]). + +-export([ + no_var_test/1, + with_hip15_vars_test/1, + with_hip17_vars_test/1, + comparison_test/1 +]). + +all() -> + [ + no_var_test, + with_hip15_vars_test, + with_hip17_vars_test, + comparison_test + ]. + +%%-------------------------------------------------------------------- +%% TEST SUITE SETUP +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + {ok, StorePid} = blockchain_test_reward_store:start(), + [{store, StorePid} | Config]. + +%%-------------------------------------------------------------------- +%% TEST SUITE TEARDOWN +%%-------------------------------------------------------------------- + +end_per_suite(_Config) -> + blockchain_test_reward_store:stop(), + ok. + +%%-------------------------------------------------------------------- +%% TEST CASE SETUP +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + Config0 = blockchain_ct_utils:init_base_dir_config(?MODULE, TestCase, Config), + Balance = 5000, + {ok, Sup, {PrivKey, PubKey}, Opts} = test_utils:init(?config(base_dir, Config0)), + + ExtraVars = + case TestCase of + with_hip15_vars_test -> + hip15_vars(); + with_hip17_vars_test -> + maps:merge(hip15_vars(), hip17_vars()); + _ -> + #{} + end, + + {ok, GenesisMembers, _GenesisBlock, ConsensusMembers, Keys} = + test_utils:init_chain(Balance, {PrivKey, PubKey}, true, ExtraVars), + + Chain = blockchain_worker:blockchain(), + Swarm = blockchain_swarm:swarm(), + N = length(ConsensusMembers), + + % Check ledger to make sure everyone has the right balance + Ledger = blockchain:ledger(Chain), + + %% Add hexes to the ledger + LedgerC = blockchain_ledger_v1:new_context(Ledger), + ok = blockchain:bootstrap_hexes(LedgerC), + ok = blockchain_ledger_v1:bootstrap_h3dex(LedgerC), + ok = blockchain_ledger_v1:commit_context(LedgerC), + ok = blockchain_ledger_v1:compact(Ledger), + + Entries = blockchain_ledger_v1:entries(Ledger), + _ = lists:foreach( + fun(Entry) -> + Balance = blockchain_ledger_entry_v1:balance(Entry), + 0 = blockchain_ledger_entry_v1:nonce(Entry) + end, + maps:values(Entries) + ), + + meck:new(blockchain_txn_rewards_v1, [passthrough]), + meck:new(blockchain_txn_poc_receipts_v1, [passthrough]), + + [ + {balance, Balance}, + {sup, Sup}, + {pubkey, PubKey}, + {privkey, PrivKey}, + {opts, Opts}, + {chain, Chain}, + {swarm, Swarm}, + {n, N}, + {consensus_members, ConsensusMembers}, + {genesis_members, GenesisMembers}, + {tc_name, TestCase}, + Keys + | Config0 + ]. + +%%-------------------------------------------------------------------- +%% TEST CASE TEARDOWN +%%-------------------------------------------------------------------- + +end_per_testcase(_TestCase, Config) -> + meck:unload(blockchain_txn_rewards_v1), + meck:unload(blockchain_txn_poc_receipts_v1), + meck:unload(), + Sup = ?config(sup, Config), + % Make sure blockchain saved on file = in memory + case erlang:is_process_alive(Sup) of + true -> + true = erlang:exit(Sup, normal), + ok = test_utils:wait_until(fun() -> false =:= erlang:is_process_alive(Sup) end); + false -> + ok + end, + ok. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +no_var_test(Config) -> + Witnesses = [b, c, e, f, g], + run_test(Witnesses, Config). + +with_hip15_vars_test(Config) -> + %% - We have gateways: [a, b, c, d, e, f, g, h, i, j, k] + %% - We'll make a poc receipt txn by hand, without any validation + %% - We'll also consider that all witnesses are legit (legit_witnesses) + %% - The poc transaction will have path like so: a -> d -> h + %% - For a -> d; [b, c, e, f, g] will all be witnesses, their rewards should get scaled + %% - For d -> h; 0 witnesses + Witnesses = [b, c, e, f, g], + run_test(Witnesses, Config). + +with_hip17_vars_test(Config) -> + %% - We have gateways: [a, b, c, d, e, f, g, h, i, j, k] + %% - We'll make a poc receipt txn by hand, without any validation + %% - We'll also consider that all witnesses are legit (legit_witnesses) + %% - The poc transaction will have path like so: a -> d -> h + %% - For a -> d; [b, c, e, f, g] will all be witnesses, their rewards should get scaled + %% - For d -> h; 0 witnesses + Witnesses = [b, c, e, f, g], + run_test(Witnesses, Config). + +comparison_test(_Config) -> + %% Aggregate rewards from no_var_test + NoVarChallengeeRewards = blockchain_test_reward_store:fetch(no_var_test_challengee_rewards), + NoVarWitnessRewards = blockchain_test_reward_store:fetch(no_var_test_witness_rewards), + + TotalNoVarChallengeeRewards = lists:sum(maps:values(NoVarChallengeeRewards)), + TotalNoVarWitnessRewards = lists:sum(maps:values(NoVarWitnessRewards)), + + FractionalNoVarChallengeeRewards = maps:map(fun(_, V) -> V / TotalNoVarChallengeeRewards end, NoVarChallengeeRewards), + FractionalNoVarWitnessRewards = maps:map(fun(_, V) -> V / TotalNoVarWitnessRewards end, NoVarWitnessRewards), + + %% Aggregate rewards from hip15 test + Hip15ChallengeeRewards = blockchain_test_reward_store:fetch(with_hip15_vars_test_challengee_rewards), + Hip15WitnessRewards = blockchain_test_reward_store:fetch(with_hip15_vars_test_witness_rewards), + + TotalHip15ChallengeeRewards = lists:sum(maps:values(Hip15ChallengeeRewards)), + TotalHip15WitnessRewards = lists:sum(maps:values(Hip15WitnessRewards)), + + FractionalHip15ChallengeeRewards = maps:map(fun(_, V) -> V / TotalHip15ChallengeeRewards end, Hip15ChallengeeRewards), + FractionalHip15WitnessRewards = maps:map(fun(_, V) -> V / TotalHip15WitnessRewards end, Hip15WitnessRewards), + + %% Aggregate rewards from hip17 test + Hip17ChallengeeRewards = blockchain_test_reward_store:fetch(with_hip17_vars_test_challengee_rewards), + Hip17WitnessRewards = blockchain_test_reward_store:fetch(with_hip17_vars_test_witness_rewards), + + TotalHip17ChallengeeRewards = lists:sum(maps:values(Hip17ChallengeeRewards)), + TotalHip17WitnessRewards = lists:sum(maps:values(Hip17WitnessRewards)), + + FractionalHip17ChallengeeRewards = maps:map(fun(_, V) -> V / TotalHip17ChallengeeRewards end, Hip17ChallengeeRewards), + FractionalHip17WitnessRewards = maps:map(fun(_, V) -> V / TotalHip17WitnessRewards end, Hip17WitnessRewards), + + ct:pal("NoVarChallengeeRewards: ~p", [NoVarChallengeeRewards]), + ct:pal("FractionalNoVarChallengeeRewards: ~p", [FractionalNoVarChallengeeRewards]), + + ct:pal("NoVarWitnessRewards: ~p", [NoVarWitnessRewards]), + ct:pal("FractionalNoVarWitnessRewards: ~p", [FractionalNoVarWitnessRewards]), + + ct:pal("Hip15ChallengeeRewards: ~p", [Hip15ChallengeeRewards]), + ct:pal("FractionalHip15ChallengeeRewards: ~p", [FractionalHip15ChallengeeRewards]), + + ct:pal("Hip15WitnessRewards: ~p", [Hip15WitnessRewards]), + ct:pal("FractionalHip15WitnessRewards: ~p", [FractionalHip15WitnessRewards]), + + ct:pal("Hip17ChallengeeRewards: ~p", [Hip17ChallengeeRewards]), + ct:pal("FractionalHip17ChallengeeRewards: ~p", [FractionalHip17ChallengeeRewards]), + + ct:pal("Hip17WitnessRewards: ~p", [Hip17WitnessRewards]), + ct:pal("FractionalHip17WitnessRewards: ~p", [FractionalHip17WitnessRewards]), + + %% TODO: Actually assert some invariant and compare the three results + + ok. + +%%-------------------------------------------------------------------- +%% HELPER +%%-------------------------------------------------------------------- + +run_test(Witnesses, Config) -> + ct:pal("Config: ~p", [Config]), + BaseDir = ?config(base_dir, Config), + ConsensusMembers = ?config(consensus_members, Config), + BaseDir = ?config(base_dir, Config), + Chain = ?config(chain, Config), + TCName = ?config(tc_name, Config), + Store = ?config(store, Config), + ct:pal("store: ~p", [Store]), + + Ledger = blockchain:ledger(Chain), + Vars = blockchain_ledger_v1:snapshot_vars(Ledger), + ct:pal("Vars: ~p", [Vars]), + + AG = blockchain_ledger_v1:active_gateways(Ledger), + + GatewayAddrs = lists:sort(maps:keys(AG)), + + AllGws = [a, b, c, d, e, f, g, h, i, j, k], + + %% For crosscheck + GatewayNameMap = lists:foldl( + fun({Letter, A}, Acc) -> + maps:put(blockchain_utils:addr2name(A), Letter, Acc) + end, + #{}, + lists:zip(AllGws, GatewayAddrs) + ), + + %% For crosscheck + GatewayLocMap = lists:foldl( + fun(A, Acc) -> + {ok, Gw} = blockchain_ledger_v1:find_gateway_info(A, Ledger), + GwLoc = blockchain_ledger_gateway_v2:location(Gw), + maps:put(blockchain_utils:addr2name(A), GwLoc, Acc) + end, + #{}, + GatewayAddrs + ), + + %% For crosscheck + GatewayLetterToAddrMap = lists:foldl( + fun({Letter, A}, Acc) -> + maps:put(Letter, A, Acc) + end, + #{}, + lists:zip(AllGws, GatewayAddrs) + ), + + Challenger = maps:get(k, GatewayLetterToAddrMap), + + GwA = maps:get(a, GatewayLetterToAddrMap), + GwD = maps:get(d, GatewayLetterToAddrMap), + GwH = maps:get(h, GatewayLetterToAddrMap), + + Rx1 = blockchain_poc_receipt_v1:new( + GwA, + 1000, + 10, + "first_rx", + p2p + ), + Rx2 = blockchain_poc_receipt_v1:new( + GwD, + 1000, + 10, + "second_rx", + radio + ), + Rx3 = blockchain_poc_receipt_v1:new( + GwH, + 1000, + 10, + "third_rx", + radio + ), + + ct:pal("Rx1: ~p", [Rx1]), + ct:pal("Rx2: ~p", [Rx2]), + ct:pal("Rx3: ~p", [Rx3]), + + ConstructedWitnesses = lists:foldl( + fun(W, Acc) -> + WitnessGw = maps:get(W, GatewayLetterToAddrMap), + Witness = blockchain_poc_witness_v1:new( + WitnessGw, + 1001, + 10, + crypto:strong_rand_bytes(32), + 9.8, + 915.2, + 10, + <<"data_rate">> + ), + [Witness | Acc] + end, + [], + Witnesses + ), + + ct:pal("ConstructedWitnesses: ~p", [ConstructedWitnesses]), + + %% We'll consider all the witnesses to be "good quality" for the sake of testing + meck:expect( + blockchain_txn_rewards_v1, + legit_witnesses, + fun(_, _, _, _, _, _) -> + ConstructedWitnesses + end + ), + meck:expect(blockchain_txn_poc_receipts_v1, absorb, fun(_, _) -> ok end), + meck:expect(blockchain_txn_poc_receipts_v1, valid_witnesses, fun(_, _, _) -> + ConstructedWitnesses + end), + meck:expect(blockchain_txn_poc_receipts_v1, good_quality_witnesses, fun(_, _) -> + ConstructedWitnesses + end), + meck:expect(blockchain_txn_poc_receipts_v1, get_channels, fun(_, _) -> + {ok, lists:seq(1, 11)} + end), + + P1 = blockchain_poc_path_element_v1:new(GwA, Rx1, []), + P2 = blockchain_poc_path_element_v1:new(GwD, Rx2, ConstructedWitnesses), + P3 = blockchain_poc_path_element_v1:new(GwH, Rx3, []), + + ct:pal("P1: ~p", [P1]), + ct:pal("P2: ~p", [P2]), + ct:pal("P3: ~p", [P3]), + + Txn = blockchain_txn_poc_receipts_v1:new( + Challenger, + <<"secret">>, + <<"onion_key_hash">>, + <<"block_hash">>, + [P1, P2, P3] + ), + ct:pal("Txn: ~p", [Txn]), + + %% Construct a block for the poc receipt txn WITHOUT validation + {ok, Block2} = test_utils:create_block(ConsensusMembers, [Txn], #{}, false), + ct:pal("Block2: ~p", [Block2]), + _ = blockchain_gossip_handler:add_block(Block2, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 2}, blockchain:height(Chain)), + + %% Empty block + {ok, Block3} = test_utils:create_block(ConsensusMembers, []), + ct:pal("Block3: ~p", [Block3]), + _ = blockchain_gossip_handler:add_block(Block3, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 3}, blockchain:height(Chain)), + + %% Calculate rewards by hand + Start = 1, + End = 3, + {ok, Rewards} = blockchain_txn_rewards_v1:calculate_rewards(Start, End, Chain), + + ChallengeesRewards = lists:filter( + fun(R) -> + blockchain_txn_reward_v1:type(R) == poc_challengees + end, + Rewards + ), + + WitnessRewards = lists:filter( + fun(R) -> + blockchain_txn_reward_v1:type(R) == poc_witnesses + end, + Rewards + ), + + ChallengeesRewardsMap = + lists:foldl( + fun(R, Acc) -> + maps:put( + maps:get( + blockchain_utils:addr2name(blockchain_txn_reward_v1:gateway(R)), + GatewayNameMap + ), + blockchain_txn_reward_v1:amount(R), + Acc + ) + end, + #{}, + ChallengeesRewards + ), + + WitnessRewardsMap = + lists:foldl( + fun(R, Acc) -> + maps:put( + maps:get( + blockchain_utils:addr2name(blockchain_txn_reward_v1:gateway(R)), + GatewayNameMap + ), + blockchain_txn_reward_v1:amount(R), + Acc + ) + end, + #{}, + WitnessRewards + ), + + %% Theoretically, gateways J, K should have higher witness rewards than B, C, E, F, G, I + ct:pal("GatewayNameMap: ~p", [GatewayNameMap]), + ct:pal("GatewayLocMap: ~p", [GatewayLocMap]), + ct:pal("ChallengeesRewardsMap: ~p", [ChallengeesRewardsMap]), + ct:pal("WitnessRewardsMap: ~p", [WitnessRewardsMap]), + + ok = blockchain_test_reward_store:insert( + list_to_atom(atom_to_list(TCName) ++ "_witness_rewards"), + WitnessRewardsMap + ), + ok = blockchain_test_reward_store:insert( + list_to_atom(atom_to_list(TCName) ++ "_challengee_rewards"), + ChallengeesRewardsMap + ), + + ok. + +hip15_vars() -> + #{ + %% configured on chain + ?poc_version => 9, + ?reward_version => 5, + %% new hip15 vars for testing + ?poc_reward_decay_rate => 0.8, + ?witness_redundancy => 4 + }. + +hip17_vars() -> + #{ + ?hip17_res_0 => <<"2,100000,100000">>, + ?hip17_res_1 => <<"2,100000,100000">>, + ?hip17_res_2 => <<"2,100000,100000">>, + ?hip17_res_3 => <<"2,100000,100000">>, + ?hip17_res_4 => <<"1,250,800">>, + ?hip17_res_5 => <<"1,100,400">>, + ?hip17_res_6 => <<"1,25,100">>, + ?hip17_res_7 => <<"2,5,20">>, + ?hip17_res_8 => <<"2,1,4">>, + ?hip17_res_9 => <<"2,1,2">>, + ?hip17_res_10 => <<"2,1,1">>, + ?hip17_res_11 => <<"2,100000,100000">>, + ?hip17_res_12 => <<"2,100000,100000">>, + ?density_tgt_res => 8 + }. diff --git a/test/blockchain_reward_perf_SUITE.erl b/test/blockchain_reward_perf_SUITE.erl new file mode 100644 index 0000000000..4548e49d15 --- /dev/null +++ b/test/blockchain_reward_perf_SUITE.erl @@ -0,0 +1,151 @@ +%%%------------------------------------------------------------------- +%%% @author Evan Vigil-McClanahan +%%% @copyright (C) 2020, Evan Vigil-McClanahan +%%% @doc +%%% +%%% @end +%%% Created : 1 Dec 2020 by Evan Vigil-McClanahan +%%%------------------------------------------------------------------- +-module(blockchain_reward_perf_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include("blockchain_vars.hrl"). + + +-export([ + suite/0, + all/0, + init_per_suite/1, + end_per_suite/1, + init_per_testcase/2, + end_per_testcase/2 + ]). + +-export([reward_perf_test/1]). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,90}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> term() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + {ok, _} = application:ensure_all_started(lager), + + {ok, Dir} = file:get_cwd(), + PrivDir = filename:join([Dir, "priv"]), + NewDir = PrivDir ++ "/ledger/", + ok = filelib:ensure_dir(NewDir), + + os:cmd("wget https://blockchain-core.s3-us-west-1.amazonaws.com/snap-591841"), + + Filename = Dir ++ "/snap-591841", + + {ok, BinSnap} = file:read_file(Filename), + + {ok, Snapshot} = blockchain_ledger_snapshot_v1:deserialize(BinSnap), + SHA = blockchain_ledger_snapshot_v1:hash(Snapshot), + + {ok, _Pid} = blockchain_score_cache:start_link(), + + {ok, BinGen} = file:read_file("../../../../test/genesis"), + GenesisBlock = blockchain_block:deserialize(BinGen), + {ok, Chain} = blockchain:new(NewDir, GenesisBlock, blessed_snapshot, undefined), + + {ok, Ledger1} = blockchain_ledger_snapshot_v1:import(Chain, SHA, Snapshot), + {ok, Height} = blockchain_ledger_v1:current_height(Ledger1), + + ct:pal("loaded ledger at height ~p", [Height]), + + [{chain, Chain} | Config]. + +end_per_testcase(_TestCase, _Config) -> + blockchain_score_cache:stop(), + ok. + +all() -> + []. + +reward_perf_test(Config) -> + Chain = ?config(chain, Config), + Ledger = blockchain:ledger(Chain), + + {ok, Height} = blockchain_ledger_v1:current_height(Ledger), + + {Time, _} = + timer:tc( + fun() -> + blockchain_txn_rewards_v1:calculate_rewards(Height - 30, Height, Chain) + end), + ct:pal("basic calc took: ~p ms", [Time div 1000]), + + {Time2, _} = + timer:tc( + fun() -> + blockchain_txn_rewards_v1:calculate_rewards(Height - 30, Height, Chain) + end), + ct:pal("basic calc 2 took: ~p ms", [Time2 div 1000]), + + Vars = maps:merge(hip15_vars(), hip17_vars()), + Ledger1 = blockchain_ledger_v1:new_context(Ledger), + ok = blockchain_ledger_v1:vars(Vars, [], Ledger1), + blockchain_ledger_v1:commit_context(Ledger1), + + {Time3, _} = + timer:tc( + fun() -> + blockchain_txn_rewards_v1:calculate_rewards(Height - 30, Height, Chain) + end), + ct:pal("hip 17 calc took: ~p ms", [Time3 div 1000]), + + + ok. + + +hip15_vars() -> + #{ + %% configured on chain + ?poc_version => 9, + ?reward_version => 5, + %% new hip15 vars for testing + ?poc_reward_decay_rate => 0.8, + ?witness_redundancy => 4 + }. + +hip17_vars() -> + #{ + ?hip17_res_0 => <<"2,100000,100000">>, + ?hip17_res_1 => <<"2,100000,100000">>, + ?hip17_res_2 => <<"2,100000,100000">>, + ?hip17_res_3 => <<"2,100000,100000">>, + ?hip17_res_4 => <<"1,250,800">>, + ?hip17_res_5 => <<"1,100,400">>, + ?hip17_res_6 => <<"1,25,100">>, + ?hip17_res_7 => <<"2,5,20">>, + ?hip17_res_8 => <<"2,1,4">>, + ?hip17_res_9 => <<"2,1,2">>, + ?hip17_res_10 => <<"2,1,1">>, + ?hip17_res_11 => <<"2,100000,100000">>, + ?hip17_res_12 => <<"2,100000,100000">>, + ?density_tgt_res => 8 + }. From 85123a06b3e518fb5a50679e38277793a63f3bbc Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Wed, 2 Dec 2020 16:19:44 -0600 Subject: [PATCH 05/18] Fixes to hex API and related chain vars - Start at k-ring of 2 - Fix exporting scale test data - Cleanup now unneeded debugging - Update limit function - Define lower resolution bound for rewards - Use chain var for lower bound res - Update scale_test to actually do cross checking - Update hip17 reward suite to run again --- src/blockchain_hex.erl | 69 +++++++++++++++---- .../v1/blockchain_txn_rewards_v1.erl | 11 ++- test/blockchain_hex_SUITE.erl | 35 +++++++--- test/blockchain_reward_hip17_SUITE.erl | 5 ++ 4 files changed, 94 insertions(+), 26 deletions(-) diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl index 96a1e30ac7..2ba107beaf 100644 --- a/src/blockchain_hex.erl +++ b/src/blockchain_hex.erl @@ -1,37 +1,54 @@ -module(blockchain_hex). --export([var_map/1, scale/3]). +-export([var_map/1, scale/3, destroy_memoization/0]). -ifdef(TEST). -export([densities/3]). -endif. -include("blockchain_vars.hrl"). - -include_lib("common_test/include/ct.hrl"). +-define(MEMO_TBL, '__blockchain_hex_memoization_tbl'). +-define(ETS_OPTS, [named_table, public]). + -type density_map() :: #{h3:h3_index() => pos_integer()}. -type densities() :: {UnclippedDensities :: density_map(), ClippedDensities :: density_map()}. -type var_map() :: #{0..12 => map()}. -type locations() :: #{h3:h3_index() => [libp2p_crypto:pubkey_bin(), ...]}. -type h3_indices() :: [h3:h3_index()]. +-export_type([var_map/0]). + %%-------------------------------------------------------------------- %% Public functions %%-------------------------------------------------------------------- +-spec destroy_memoization() -> true. +%% @doc This call will destroy the memoization context used during a rewards +%% calculation. +destroy_memoization() -> ets:delete(?MEMO_TBL). + -spec scale( Location :: h3:h3_index(), VarMap :: var_map(), Ledger :: blockchain_ledger_v1:ledger() ) -> float(). +%% @doc Given a hex location, return the rewards scaling factor. This call is +%% memoized. scale(Location, VarMap, Ledger) -> - {UnclippedDensities, ClippedDensities} = densities(Location, VarMap, Ledger), - maps:get(Location, ClippedDensities) / maps:get(Location, UnclippedDensities). + case lookup(Location) of + {ok, Scale} -> Scale; + not_found -> + memoize(Location, do_scale(Location, VarMap, Ledger)) + end. + %% TODO: This ought to be stored in the ledger because it won't change much? ever? %% after it's been computed. Seems dumb to calculate it every single time we pay %% out rewards. -spec var_map(Ledger :: blockchain_ledger_v1:ledger()) -> {error, any()} | {ok, var_map()}. +%% @doc This function returns a map of hex resolutions mapped to hotspot density targets and +%% maximums. These numbers are used during PoC witness and challenge rewards calculations. var_map(Ledger) -> ResolutionVars = [ ?hip17_res_0, @@ -79,13 +96,40 @@ var_map(Ledger) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +-spec lookup(Key :: term()) -> {ok, Result :: term()} | not_found. +lookup(Key) -> + try + case ets:lookup(?MEMO_TBL, Key) of + [{_Key, Res}] -> {ok, Res}; + [] -> not_found + end + catch + %% if the table doesn't exist yet, create it and return `not_found' + _:badarg -> + _Name = ets:new(?MEMO_TBL, ?ETS_OPTS), + not_found + end. + +-spec memoize(Key :: term(), Result :: term()) -> Result :: term(). +memoize(Key, Result) -> + true = ets:insert(?MEMO_TBL, {Key, Result}), + Result. + +-spec do_scale( + Location :: h3:h3_index(), + VarMap :: var_map(), + Ledger :: blockchain_ledger_v1:ledger() ) -> float(). +do_scale(Location, VarMap, Ledger) -> + {UnclippedDensities, ClippedDensities} = densities(Location, VarMap, Ledger), + maps:get(Location, ClippedDensities) / maps:get(Location, UnclippedDensities). + -spec densities( H3Index :: h3:h3_index(), VarMap :: var_map(), Ledger :: blockchain_ledger_v1:ledger() ) -> densities(). densities(H3Index, VarMap, Ledger) -> - Locations = blockchain_ledger_v1:lookup_gateways_from_hex(h3:k_ring(H3Index, 1), Ledger), + Locations = blockchain_ledger_v1:lookup_gateways_from_hex(h3:k_ring(H3Index, 2), Ledger), %% Calculate clipped and unclipped densities densities(H3Index, VarMap, Locations, Ledger). @@ -102,7 +146,6 @@ densities(H3Root, VarMap, Locations, Ledger) -> _ -> UpperBoundRes = lists:max([h3:get_resolution(H3) || H3 <- maps:keys(Locations)]), LowerBoundRes = h3:get_resolution(H3Root), - ct:pal("UpperBoundRes: ~p, LowerBoundRes: ~p", [UpperBoundRes, LowerBoundRes]), [Head | Tail] = lists:seq(UpperBoundRes, LowerBoundRes, -1), @@ -155,9 +198,6 @@ build_densities(H3Root, Ledger, VarMap, ChildHexes, {UAcc, Acc}, [Res | Tail]) - fun(ThisResHex, Acc3) -> OccupiedCount = occupied_count(DensityTarget, ThisResHex, UD), Limit = limit(Res, VarMap, OccupiedCount), - - ct:pal("Limit: ~p, OccupiedCount: ~p", [Limit, OccupiedCount]), - maps:put(ThisResHex, min(Limit, maps:get(ThisResHex, M0)), Acc3) end, M0, @@ -172,11 +212,12 @@ build_densities(H3Root, Ledger, VarMap, ChildHexes, {UAcc, Acc}, [Res | Tail]) - OccupiedCount :: non_neg_integer() ) -> non_neg_integer(). limit(Res, VarMap, OccupiedCount) -> - min( - maps:get(max, maps:get(Res, VarMap)), - maps:get(tgt, maps:get(Res, VarMap)) * - max((OccupiedCount - maps:get(n, maps:get(Res, VarMap))), 1) - ). + VarAtRes = maps:get(Res, VarMap), + DensityMax = maps:get(max, VarAtRes), + DensityTgt = maps:get(tgt, VarAtRes), + N = maps:get(n, VarAtRes), + Max = max((OccupiedCount - N), 1), + min(DensityMax, DensityTgt * Max). -spec occupied_count( DensityTarget :: 0..12, diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index 19083b994a..e035737721 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -194,6 +194,10 @@ calculate_rewards(Start, End, Chain) -> [ConsensusRewards, SecuritiesRewards, POCChallengersRewards, POCChallengeesRewards, POCWitnessesRewards, DCRewards] ), + %% we are only keeping hex density calculations memoized for a single + %% rewards transaction calculation, then we discard that work and avoid + %% cache invalidation issues. + true = blockchain_hex:destroy_memoization(), {ok, Result} end. @@ -781,7 +785,7 @@ poc_witnesses_rewards(Transactions, Acc1, ValidWitnesses ); - _D -> + D -> %% new (HIP17) lists:foldl( fun(WitnessRecord, Map) -> @@ -791,6 +795,7 @@ poc_witnesses_rewards(Transactions, WitnessLoc = blockchain_ledger_gateway_v2:location(WitnessGw), RxScale = blockchain_hex:scale(WitnessLoc, VarMap, + D, Ledger), lager:info("WitnessGw: ~p, RxScale: ~p", [blockchain_utils:addr2name(Witness), RxScale]), @@ -1063,8 +1068,8 @@ maybe_calc_tx_scale(Challengee, case {DensityTgtRes, ChallengeeLoc} of {undefined, _} -> 1.0; {_, undefined} -> 1.0; - {_D, Loc} -> - TxScale = blockchain_hex:scale(Loc, VarMap, Ledger), + {D, Loc} -> + TxScale = blockchain_hex:scale(Loc, VarMap, D, Ledger), lager:info("Challengee: ~p, RxScale: ~p", [blockchain_utils:addr2name(Challengee), TxScale]), TxScale diff --git a/test/blockchain_hex_SUITE.erl b/test/blockchain_hex_SUITE.erl index 63aab00786..492549d36a 100644 --- a/test/blockchain_hex_SUITE.erl +++ b/test/blockchain_hex_SUITE.erl @@ -114,7 +114,7 @@ init_per_suite(Config) -> %% Check that the vars are correct, one is enough... {ok, VarMap} = blockchain_hex:var_map(Ledger), ct:pal("var_map: ~p", [VarMap]), - #{ n := 1, tgt := 250, max := 800} = maps:get(4, VarMap), + #{n := 1, tgt := 250, max := 800} = maps:get(4, VarMap), [ {ledger, Ledger}, @@ -185,7 +185,8 @@ full_known_values_test(Config) -> ] ), - UnclippedValue == maps:get(Hex, UnclippedDensities) andalso ClippedValue == maps:get(Hex, ClippedDensities) + UnclippedValue == maps:get(Hex, UnclippedDensities) andalso + ClippedValue == maps:get(Hex, ClippedDensities) end end, List @@ -241,12 +242,24 @@ known_differences_test(Config) -> scale_test(Config) -> Ledger = ?config(ledger, Config), - Another = h3:from_string("8c2836152804dff"), + VarMap = ?config(var_map, Config), + TargetResolutions = lists:seq(3, 10), + KnownHex = h3:from_string("8c2836152804dff"), - Scale = blockchain_hex:scale(Another, Ledger), - ct:pal("Res: ~p, Scale: ~p", [h3:get_resolution(Another), Scale]), + Result = lists:foldl( + fun(LowerBoundRes, Acc) -> + Scale = blockchain_hex:scale(KnownHex, VarMap, LowerBoundRes, Ledger), + ct:pal("LowerBoundRes: ~p, Scale: ~p", [LowerBoundRes, Scale]), + blockchain_hex:destroy_memoization(), + maps:put(LowerBoundRes, Scale, Acc) + end, + #{}, + TargetResolutions + ), - %% TODO: Assert checks from the python model + ct:pal("Result: ~p", [Result]), + + "0.18" = io_lib:format("~.2f", [maps:get(8, Result)]), ok. @@ -273,6 +286,7 @@ h3dex_test(Config) -> export_scale_test(Config) -> Ledger = ?config(ledger, Config), + VarMap = ?config(var_map, Config), %% A list of possible density target resolution we'd output the scales at DensityTargetResolutions = lists:seq(3, 10), @@ -282,6 +296,7 @@ export_scale_test(Config) -> ok = export_scale_data( Ledger, + VarMap, DensityTargetResolutions, GatewaysWithLocs ), @@ -328,15 +343,17 @@ gateways_with_locs(Ledger) -> AG ). -export_scale_data(Ledger, DensityTargetResolutions, GatewaysWithLocs) -> +export_scale_data(Ledger, VarMap, DensityTargetResolutions, GatewaysWithLocs) -> %% Calculate scale at each density target res for eventual comparison lists:foreach( - fun(TargetRes) -> + fun(LowerBoundRes) -> %% Export scale data for every single gateway to a gps file Scales = lists:foldl( fun({GwName, Loc}, Acc) -> Scale = blockchain_hex:scale( Loc, + VarMap, + LowerBoundRes, Ledger ), [{GwName, Loc, Scale} | Acc] @@ -345,7 +362,7 @@ export_scale_data(Ledger, DensityTargetResolutions, GatewaysWithLocs) -> GatewaysWithLocs ), - Fname = "/tmp/scale_" ++ integer_to_list(TargetRes), + Fname = "/tmp/scale_" ++ integer_to_list(LowerBoundRes), ok = export_gps_file(Fname, Scales) end, DensityTargetResolutions diff --git a/test/blockchain_reward_hip17_SUITE.erl b/test/blockchain_reward_hip17_SUITE.erl index 8d275ebd9c..df60430a17 100644 --- a/test/blockchain_reward_hip17_SUITE.erl +++ b/test/blockchain_reward_hip17_SUITE.erl @@ -91,6 +91,7 @@ init_per_testcase(TestCase, Config) -> meck:new(blockchain_txn_rewards_v1, [passthrough]), meck:new(blockchain_txn_poc_receipts_v1, [passthrough]), + meck:new(blockchain_hex, [passthrough]), [ {balance, Balance}, @@ -115,6 +116,7 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(_TestCase, Config) -> meck:unload(blockchain_txn_rewards_v1), meck:unload(blockchain_txn_poc_receipts_v1), + meck:unload(blockchain_hex), meck:unload(), Sup = ?config(sup, Config), % Make sure blockchain saved on file = in memory @@ -332,6 +334,9 @@ run_test(Witnesses, Config) -> meck:expect(blockchain_txn_poc_receipts_v1, get_channels, fun(_, _) -> {ok, lists:seq(1, 11)} end), + meck:expect(blockchain_hex, destroy_memoization, fun() -> + true + end), P1 = blockchain_poc_path_element_v1:new(GwA, Rx1, []), P2 = blockchain_poc_path_element_v1:new(GwD, Rx2, ConstructedWitnesses), From 52cb76159d0e274cb6c80fb043a05c6e983dce18 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Thu, 3 Dec 2020 09:54:59 -0800 Subject: [PATCH 06/18] Use challengee scale for witness rewards --- rebar.config | 2 +- rebar.lock | 2 +- .../v1/blockchain_txn_rewards_v1.erl | 32 ++++++++++++------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/rebar.config b/rebar.config index ed788ac8e0..682b77eb21 100644 --- a/rebar.config +++ b/rebar.config @@ -34,7 +34,7 @@ {base64url, "1.0.1"}, {libp2p, ".*", {git, "https://github.com/helium/erlang-libp2p.git", {branch, "master"}}}, {clique, ".*", {git, "https://github.com/helium/clique.git", {branch, "develop"}}}, - {h3, ".*", {git, "https://github.com/helium/erlang-h3.git", {branch, "adt/res0_indexes"}}}, + {h3, ".*", {git, "https://github.com/helium/erlang-h3.git", {branch, "master"}}}, {erl_angry_purple_tiger, ".*", {git, "https://github.com/helium/erl_angry_purple_tiger.git", {branch, "master"}}}, {erlang_stats, ".*", {git, "https://github.com/helium/erlang-stats.git", {branch, "master"}}}, {e2qc, ".*", {git, "https://github.com/project-fifo/e2qc", {branch, "master"}}}, diff --git a/rebar.lock b/rebar.lock index b4bd23201b..931d0af8cb 100644 --- a/rebar.lock +++ b/rebar.lock @@ -37,7 +37,7 @@ {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1}, {<<"h3">>, {git,"https://github.com/helium/erlang-h3.git", - {ref,"040cf9226699e0a1f85b14cad4dd924e5618578e"}}, + {ref,"8541da45596549e36bdbf82dcb77f19c8608e9d4"}}, 0}, {<<"helium_proto">>, {git,"https://github.com/helium/proto.git", diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index e035737721..78dab2f176 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -789,18 +789,26 @@ poc_witnesses_rewards(Transactions, %% new (HIP17) lists:foldl( fun(WitnessRecord, Map) -> - Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), - %% A witness must be on the ledger AND have a location, so this should be safe - {ok, WitnessGw} = blockchain_ledger_v1:find_gateway_info(Witness, Ledger), - WitnessLoc = blockchain_ledger_gateway_v2:location(WitnessGw), - RxScale = blockchain_hex:scale(WitnessLoc, - VarMap, - D, - Ledger), - lager:info("WitnessGw: ~p, RxScale: ~p", [blockchain_utils:addr2name(Witness), - RxScale]), - I = maps:get(Witness, Map, 0), - maps:put(Witness, I+(ToAdd*RxScale), Map) + Challengee = blockchain_poc_path_element_v1:challengee(Elem), + case blockchain_ledger_v1:find_gateway_info(Challengee, Ledger) of + {ok, ChallengeeGw} -> + case blockchain_ledger_gateway_v2:location(ChallengeeGw) of + undefined -> + Map; + ChallengeeLoc -> + Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), + %% The witnesses get scaled by the value of their transmitters + RxScale = blockchain_hex:scale(ChallengeeLoc, + VarMap, + D, + Ledger), + lager:info("WitnessGw: ~p, RxScale: ~p", [blockchain_utils:addr2name(Witness), RxScale]), + I = maps:get(Witness, Map, 0), + maps:put(Witness, I+(ToAdd*RxScale), Map) + end; + _ -> + Map + end end, Acc1, ValidWitnesses From af7c0c92d866969744f0e63f78a9c5415c323fb2 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Thu, 3 Dec 2020 15:23:04 -0600 Subject: [PATCH 07/18] Add filtering for interactive gateways - Fix hex test suite - Fix interactive filtering --- include/blockchain_vars.hrl | 1 + src/blockchain_hex.erl | 42 ++++++++++++++++++- .../v1/blockchain_txn_vars_v1.erl | 4 +- test/blockchain_hex_SUITE.erl | 5 ++- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/include/blockchain_vars.hrl b/include/blockchain_vars.hrl index 2e720bedc4..75714f7df4 100644 --- a/include/blockchain_vars.hrl +++ b/include/blockchain_vars.hrl @@ -398,3 +398,4 @@ -define(hip17_res_11, hip17_res_11). -define(hip17_res_12, hip17_res_12). -define(density_tgt_res, density_tgt_res). +-define(hip17_interactivity_blocks, hip17_interactivity_blocks). diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl index 2ba107beaf..dde2663a9b 100644 --- a/src/blockchain_hex.erl +++ b/src/blockchain_hex.erl @@ -123,15 +123,32 @@ do_scale(Location, VarMap, Ledger) -> {UnclippedDensities, ClippedDensities} = densities(Location, VarMap, Ledger), maps:get(Location, ClippedDensities) / maps:get(Location, UnclippedDensities). + -spec densities( H3Index :: h3:h3_index(), VarMap :: var_map(), Ledger :: blockchain_ledger_v1:ledger() ) -> densities(). densities(H3Index, VarMap, Ledger) -> + InteractiveBlocks = case blockchain_ledger_v1:config(?hip17_interactivity_blocks, Ledger) of + {ok, V} -> V; + {error, not_found} -> 0 % XXX what should this value be? + end, Locations = blockchain_ledger_v1:lookup_gateways_from_hex(h3:k_ring(H3Index, 2), Ledger), + + Interactive = case application:get_env(blockchain, hip17_test_mode, false) of + true -> + %% HIP17 test mode, no interactive filtering + Locations; + false -> + maps:map( + fun(_K, V) -> + filter_interactive_gws(V, InteractiveBlocks, Ledger) + end, Locations) + end, + %% Calculate clipped and unclipped densities - densities(H3Index, VarMap, Locations, Ledger). + densities(H3Index, VarMap, Interactive, Ledger). -spec densities( H3Root :: h3:h3_index(), @@ -181,7 +198,7 @@ densities(H3Root, VarMap, Locations, Ledger) -> var_map(), h3_indices(), densities(), - [non_neg_integer()] + [0..15] ) -> densities(). build_densities(_H3Root, _Ledger, _VarMap, _ParentHexes, {UAcc, Acc}, []) -> {UAcc, Acc}; @@ -206,6 +223,27 @@ build_densities(H3Root, Ledger, VarMap, ChildHexes, {UAcc, Acc}, [Res | Tail]) - build_densities(H3Root, Ledger, VarMap, OccupiedHexesThisRes, {UM0, M1}, Tail). +-spec filter_interactive_gws( GWs :: [libp2p_crypto:pubkey_bin(), ...], + InteractiveBlocks :: pos_integer(), + Ledger :: blockchain_ledger_v1:ledger()) -> + [libp2p_crypto:pubkey_bin(), ...]. +%% @doc This function filters a list of gateway addresses which are considered +%% "interactive" for the purposes of HIP17 based on the last block when it +%% responded to a POC challenge compared to the current chain height. +filter_interactive_gws(GWs, InteractiveBlocks, Ledger) -> + {ok, CurrentHeight} = blockchain_ledger_v1:current_height(Ledger), + lists:filter(fun(GWAddr) -> + case blockchain_ledger_v1:find_gateway_info(GWAddr, Ledger) of + {ok, GWInfo} -> + case blockchain_ledger_gateway_v2:last_poc_challenge(GWInfo) of + undefined -> false; + LastChallenge -> + (CurrentHeight - LastChallenge) =< InteractiveBlocks + end; + {error, not_found} -> false + end + end, GWs). + -spec limit( Res :: 0..12, VarMap :: var_map(), diff --git a/src/transactions/v1/blockchain_txn_vars_v1.erl b/src/transactions/v1/blockchain_txn_vars_v1.erl index 9fb91c5e81..4e636e9b18 100644 --- a/src/transactions/v1/blockchain_txn_vars_v1.erl +++ b/src/transactions/v1/blockchain_txn_vars_v1.erl @@ -1100,6 +1100,8 @@ validate_var(?hip17_res_12, Value) -> validate_hip17_vars(Value, "hip17_res_12"); validate_var(?density_tgt_res, Value) -> validate_int(Value, "density_tgt_res", 1, 15, false); +validate_var(?hip17_interactivity_blocks, Value) -> + validate_int(Value, "hip17_interactivity_blocks", 1, 5000, false); validate_var(Var, Value) -> %% something we don't understand, crash @@ -1108,7 +1110,7 @@ validate_var(Var, Value) -> validate_hip17_vars(Value, Var) when is_binary(Value) -> case get_density_var(Value) of {error, _}=E0 -> - lager:error("unable to get densit var, reason: ~p", [E0]), + lager:error("unable to get density var, reason: ~p", [E0]), throw({error, {invalid_density_var, Var, Value}}); {ok, Res} -> case length(Res) == 3 of diff --git a/test/blockchain_hex_SUITE.erl b/test/blockchain_hex_SUITE.erl index 492549d36a..126c541b2c 100644 --- a/test/blockchain_hex_SUITE.erl +++ b/test/blockchain_hex_SUITE.erl @@ -103,6 +103,8 @@ init_per_suite(Config) -> LedgerURL = "https://blockchain-core.s3-us-west-1.amazonaws.com/ledger-586724.tar.gz", Ledger = blockchain_ct_utils:ledger(hip17_vars(), LedgerURL), + ok = application:set_env(blockchain, hip17_test_mode, true), + Ledger1 = blockchain_ledger_v1:new_context(Ledger), blockchain:bootstrap_h3dex(Ledger1), blockchain_ledger_v1:commit_context(Ledger1), @@ -322,7 +324,8 @@ hip17_vars() -> hip17_res_10 => <<"2,1,1">>, hip17_res_11 => <<"2,100000,100000">>, hip17_res_12 => <<"2,100000,100000">>, - density_tgt_res => 8 + density_tgt_res => 8, + hip17_interactivity_blocks => 1200 * 3 }. %%-------------------------------------------------------------------- From 9ac163445272b46108b6323285b2c9b05827e239 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Fri, 4 Dec 2020 11:24:10 -0800 Subject: [PATCH 08/18] Optimize scaling by using density at outermost hex --- src/blockchain_hex.erl | 30 ++++++++++++----- .../v1/blockchain_txn_poc_receipts_v1.erl | 2 +- test/blockchain_hex_SUITE.erl | 14 ++++---- test/blockchain_reward_perf_SUITE.erl | 32 +++++++++++-------- 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl index dde2663a9b..3456384cad 100644 --- a/src/blockchain_hex.erl +++ b/src/blockchain_hex.erl @@ -1,6 +1,6 @@ -module(blockchain_hex). --export([var_map/1, scale/3, destroy_memoization/0]). +-export([var_map/1, scale/4, destroy_memoization/0]). -ifdef(TEST). -export([densities/3]). @@ -26,20 +26,22 @@ -spec destroy_memoization() -> true. %% @doc This call will destroy the memoization context used during a rewards %% calculation. -destroy_memoization() -> ets:delete(?MEMO_TBL). +destroy_memoization() -> + try ets:delete(?MEMO_TBL) catch _:_ -> true end. -spec scale( Location :: h3:h3_index(), VarMap :: var_map(), + TargetRes :: 0..12, Ledger :: blockchain_ledger_v1:ledger() ) -> float(). %% @doc Given a hex location, return the rewards scaling factor. This call is %% memoized. -scale(Location, VarMap, Ledger) -> +scale(Location, VarMap, TargetRes, Ledger) -> case lookup(Location) of {ok, Scale} -> Scale; not_found -> - memoize(Location, do_scale(Location, VarMap, Ledger)) + memoize(Location, calculate_scale(Location, VarMap, TargetRes, Ledger)) end. @@ -115,13 +117,25 @@ memoize(Key, Result) -> true = ets:insert(?MEMO_TBL, {Key, Result}), Result. --spec do_scale( +-spec calculate_scale( Location :: h3:h3_index(), VarMap :: var_map(), + TargetRes :: 0..12, Ledger :: blockchain_ledger_v1:ledger() ) -> float(). -do_scale(Location, VarMap, Ledger) -> - {UnclippedDensities, ClippedDensities} = densities(Location, VarMap, Ledger), - maps:get(Location, ClippedDensities) / maps:get(Location, UnclippedDensities). +calculate_scale(Location, VarMap, TargetRes, Ledger) -> + %% hip0017 states to go from R -> 0 and take a product of the clipped(parent)/unclipped(parent) + %% however, we specify the lower bound instead of going all the way down to 0 + + R = h3:get_resolution(Location), + + %% Calculate densities at the outermost hex + OuterMostParent = h3:parent(Location, TargetRes), + {UnclippedDensities, ClippedDensities} = densities(OuterMostParent, VarMap, Ledger), + + lists:foldl(fun(Res, Acc) -> + Parent = h3:parent(Location, Res), + Acc * (maps:get(Parent, ClippedDensities) / maps:get(Parent, UnclippedDensities)) + end, 1.0, lists:seq(R, TargetRes, -1)). -spec densities( diff --git a/src/transactions/v1/blockchain_txn_poc_receipts_v1.erl b/src/transactions/v1/blockchain_txn_poc_receipts_v1.erl index c3a5f400fb..1ffcdab354 100644 --- a/src/transactions/v1/blockchain_txn_poc_receipts_v1.erl +++ b/src/transactions/v1/blockchain_txn_poc_receipts_v1.erl @@ -1269,7 +1269,6 @@ get_channels(Txn, Chain) -> PathLength = length(Path), OnionKeyHash = ?MODULE:onion_key_hash(Txn), - {ok, Head} = blockchain:head_block(Chain), BlockHash = case blockchain:config(?poc_version, blockchain:ledger(Chain)) of {ok, POCVer} when POCVer >= 10 -> @@ -1282,6 +1281,7 @@ get_channels(Txn, Chain) -> blockchain_txn_poc_request_v1:onion_key_hash(T) == OnionKeyHash end, + {ok, Head} = blockchain:head_block(Chain), blockchain:fold_chain(fun(Block, undefined) -> case blockchain_utils:find_txn(Block, RequestFilter) of [_T] -> diff --git a/test/blockchain_hex_SUITE.erl b/test/blockchain_hex_SUITE.erl index 126c541b2c..57fa5fc81c 100644 --- a/test/blockchain_hex_SUITE.erl +++ b/test/blockchain_hex_SUITE.erl @@ -249,11 +249,11 @@ scale_test(Config) -> KnownHex = h3:from_string("8c2836152804dff"), Result = lists:foldl( - fun(LowerBoundRes, Acc) -> - Scale = blockchain_hex:scale(KnownHex, VarMap, LowerBoundRes, Ledger), - ct:pal("LowerBoundRes: ~p, Scale: ~p", [LowerBoundRes, Scale]), + fun(TargetRes, Acc) -> + Scale = blockchain_hex:scale(KnownHex, VarMap, TargetRes, Ledger), + ct:pal("TargetRes: ~p, Scale: ~p", [TargetRes, Scale]), blockchain_hex:destroy_memoization(), - maps:put(LowerBoundRes, Scale, Acc) + maps:put(TargetRes, Scale, Acc) end, #{}, TargetResolutions @@ -349,14 +349,14 @@ gateways_with_locs(Ledger) -> export_scale_data(Ledger, VarMap, DensityTargetResolutions, GatewaysWithLocs) -> %% Calculate scale at each density target res for eventual comparison lists:foreach( - fun(LowerBoundRes) -> + fun(TargetRes) -> %% Export scale data for every single gateway to a gps file Scales = lists:foldl( fun({GwName, Loc}, Acc) -> Scale = blockchain_hex:scale( Loc, VarMap, - LowerBoundRes, + TargetRes, Ledger ), [{GwName, Loc, Scale} | Acc] @@ -365,7 +365,7 @@ export_scale_data(Ledger, VarMap, DensityTargetResolutions, GatewaysWithLocs) -> GatewaysWithLocs ), - Fname = "/tmp/scale_" ++ integer_to_list(LowerBoundRes), + Fname = "/tmp/scale_" ++ integer_to_list(TargetRes), ok = export_gps_file(Fname, Scales) end, DensityTargetResolutions diff --git a/test/blockchain_reward_perf_SUITE.erl b/test/blockchain_reward_perf_SUITE.erl index 4548e49d15..e1d7f87831 100644 --- a/test/blockchain_reward_perf_SUITE.erl +++ b/test/blockchain_reward_perf_SUITE.erl @@ -21,7 +21,11 @@ end_per_testcase/2 ]). --export([reward_perf_test/1]). +-export([ + reward_perf_test/1, + hip15_vars/0, + hip17_vars/0 + ]). %%-------------------------------------------------------------------- %% @spec suite() -> Info @@ -29,7 +33,7 @@ %% @end %%-------------------------------------------------------------------- suite() -> - [{timetrap,{seconds,90}}]. + [{timetrap,{seconds,200}}]. %%-------------------------------------------------------------------- %% @spec init_per_suite(Config0) -> @@ -66,6 +70,7 @@ init_per_testcase(_TestCase, Config) -> {ok, Snapshot} = blockchain_ledger_snapshot_v1:deserialize(BinSnap), SHA = blockchain_ledger_snapshot_v1:hash(Snapshot), + {ok, _GWCache} = blockchain_gateway_cache:start_link(), {ok, _Pid} = blockchain_score_cache:start_link(), {ok, BinGen} = file:read_file("../../../../test/genesis"), @@ -95,37 +100,38 @@ reward_perf_test(Config) -> {Time, _} = timer:tc( fun() -> - blockchain_txn_rewards_v1:calculate_rewards(Height - 30, Height, Chain) + {ok, _} = blockchain_txn_rewards_v1:calculate_rewards(Height - 15, Height, Chain) end), ct:pal("basic calc took: ~p ms", [Time div 1000]), - {Time2, _} = - timer:tc( - fun() -> - blockchain_txn_rewards_v1:calculate_rewards(Height - 30, Height, Chain) - end), - ct:pal("basic calc 2 took: ~p ms", [Time2 div 1000]), + %% {Time2, _} = + %% timer:tc( + %% fun() -> + %% blockchain_txn_rewards_v1:calculate_rewards(Height - 20, Height, Chain) + %% end), + %% ct:pal("basic calc 2 took: ~p ms", [Time2 div 1000]), - Vars = maps:merge(hip15_vars(), hip17_vars()), + Vars = maps:merge(blockchain_reward_perf_SUITE:hip15_vars(), blockchain_reward_perf_SUITE:hip17_vars()), Ledger1 = blockchain_ledger_v1:new_context(Ledger), ok = blockchain_ledger_v1:vars(Vars, [], Ledger1), + blockchain:bootstrap_h3dex(Ledger1), blockchain_ledger_v1:commit_context(Ledger1), {Time3, _} = timer:tc( fun() -> - blockchain_txn_rewards_v1:calculate_rewards(Height - 30, Height, Chain) + blockchain_txn_rewards_v1:calculate_rewards(Height - 15, Height, Chain) end), ct:pal("hip 17 calc took: ~p ms", [Time3 div 1000]), - + error(print), ok. hip15_vars() -> #{ %% configured on chain - ?poc_version => 9, + ?poc_version => 10, ?reward_version => 5, %% new hip15 vars for testing ?poc_reward_decay_rate => 0.8, From 46d7b90da6c3342f9b3842c936010cbf276eddbf Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Fri, 4 Dec 2020 13:48:20 -0600 Subject: [PATCH 09/18] Memoize density calculations --- src/blockchain_hex.erl | 48 ++++++++++++++++++++------ test/blockchain_reward_hip17_SUITE.erl | 1 + 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl index 3456384cad..16129f9a1d 100644 --- a/src/blockchain_hex.erl +++ b/src/blockchain_hex.erl @@ -9,7 +9,8 @@ -include("blockchain_vars.hrl"). -include_lib("common_test/include/ct.hrl"). --define(MEMO_TBL, '__blockchain_hex_memoization_tbl'). +-define(SCALE_MEMO_TBL, '__blockchain_hex_scale_memoization_tbl'). +-define(DENSITY_MEMO_TBL, '__blockchain_hex_density_memoization_tbl'). -define(ETS_OPTS, [named_table, public]). -type density_map() :: #{h3:h3_index() => pos_integer()}. @@ -27,7 +28,8 @@ %% @doc This call will destroy the memoization context used during a rewards %% calculation. destroy_memoization() -> - try ets:delete(?MEMO_TBL) catch _:_ -> true end. + try ets:delete(?SCALE_MEMO_TBL) catch _:_ -> true end, + try ets:delete(?DENSITY_MEMO_TBL) catch _:_ -> true end. -spec scale( Location :: h3:h3_index(), @@ -38,10 +40,11 @@ destroy_memoization() -> %% @doc Given a hex location, return the rewards scaling factor. This call is %% memoized. scale(Location, VarMap, TargetRes, Ledger) -> - case lookup(Location) of + case lookup(Location, ?SCALE_MEMO_TBL) of {ok, Scale} -> Scale; not_found -> - memoize(Location, calculate_scale(Location, VarMap, TargetRes, Ledger)) + memoize(?SCALE_MEMO_TBL, Location, + calculate_scale(Location, VarMap, TargetRes, Ledger)) end. @@ -98,23 +101,34 @@ var_map(Ledger) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- --spec lookup(Key :: term()) -> {ok, Result :: term()} | not_found. -lookup(Key) -> +-spec lookup(Key :: term(), + TblName :: ?SCALE_MEMO_TBL|?DENSITY_MEMO_TBL) -> + {ok, Result :: term()} | not_found. +lookup(Key, TblName) -> try - case ets:lookup(?MEMO_TBL, Key) of + case ets:lookup(TblName, Key) of [{_Key, Res}] -> {ok, Res}; [] -> not_found end catch %% if the table doesn't exist yet, create it and return `not_found' _:badarg -> - _Name = ets:new(?MEMO_TBL, ?ETS_OPTS), + _ = maybe_start(TblName), not_found end. --spec memoize(Key :: term(), Result :: term()) -> Result :: term(). -memoize(Key, Result) -> - true = ets:insert(?MEMO_TBL, {Key, Result}), +maybe_start(TblName) -> + try + _Name = ets:new(TblName, ?ETS_OPTS) + catch + _:_ -> TblName + end. + +-spec memoize(TblName :: ?SCALE_MEMO_TBL|?DENSITY_MEMO_TBL, + Key :: term(), + Result :: term()) -> Result :: term(). +memoize(TblName, Key, Result) -> + true = ets:insert(TblName, {Key, Result}), Result. -spec calculate_scale( @@ -144,6 +158,18 @@ calculate_scale(Location, VarMap, TargetRes, Ledger) -> Ledger :: blockchain_ledger_v1:ledger() ) -> densities(). densities(H3Index, VarMap, Ledger) -> + case lookup(H3Index, ?DENSITY_MEMO_TBL) of + {ok, Densities} -> Densities; + not_found -> memoize(?DENSITY_MEMO_TBL, H3Index, + calculate_densities(H3Index, VarMap, Ledger)) + end. + +-spec calculate_densities( + H3Index :: h3:h3_index(), + VarMap :: var_map(), + Ledger :: blockchain_ledger_v1:ledger() +) -> densities(). +calculate_densities(H3Index, VarMap, Ledger) -> InteractiveBlocks = case blockchain_ledger_v1:config(?hip17_interactivity_blocks, Ledger) of {ok, V} -> V; {error, not_found} -> 0 % XXX what should this value be? diff --git a/test/blockchain_reward_hip17_SUITE.erl b/test/blockchain_reward_hip17_SUITE.erl index df60430a17..ec75611067 100644 --- a/test/blockchain_reward_hip17_SUITE.erl +++ b/test/blockchain_reward_hip17_SUITE.erl @@ -52,6 +52,7 @@ init_per_testcase(TestCase, Config) -> Config0 = blockchain_ct_utils:init_base_dir_config(?MODULE, TestCase, Config), Balance = 5000, {ok, Sup, {PrivKey, PubKey}, Opts} = test_utils:init(?config(base_dir, Config0)), + ok = application:set_env(blockchain, hip17_test_mode, true), ExtraVars = case TestCase of From 797e5cd9a724869db94ae89a9175371299e2df24 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Mon, 7 Dec 2020 10:17:47 -0800 Subject: [PATCH 10/18] Add more cross checking for rewards - Fix log msg - Add another crosscheck --- .../v1/blockchain_txn_rewards_v1.erl | 4 +- test/blockchain_reward_hip17_SUITE.erl | 72 ++++++++++++++++++- test/test_utils.erl | 70 ++++++++++++++---- 3 files changed, 131 insertions(+), 15 deletions(-) diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index 78dab2f176..4b21bdb617 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -1078,8 +1078,8 @@ maybe_calc_tx_scale(Challengee, {_, undefined} -> 1.0; {D, Loc} -> TxScale = blockchain_hex:scale(Loc, VarMap, D, Ledger), - lager:info("Challengee: ~p, RxScale: ~p", [blockchain_utils:addr2name(Challengee), - TxScale]), + lager:info("Challengee: ~p, TxScale: ~p", + [blockchain_utils:addr2name(Challengee), TxScale]), TxScale end. diff --git a/test/blockchain_reward_hip17_SUITE.erl b/test/blockchain_reward_hip17_SUITE.erl index ec75611067..604ace25c5 100644 --- a/test/blockchain_reward_hip17_SUITE.erl +++ b/test/blockchain_reward_hip17_SUITE.erl @@ -64,8 +64,9 @@ init_per_testcase(TestCase, Config) -> #{} end, + {ok, GenesisMembers, _GenesisBlock, ConsensusMembers, Keys} = - test_utils:init_chain(Balance, {PrivKey, PubKey}, true, ExtraVars), + test_utils:init_chain_with_fixed_locations(Balance, {PrivKey, PubKey}, true, known_locations(), ExtraVars), Chain = blockchain_worker:blockchain(), Swarm = blockchain_swarm:swarm(), @@ -264,6 +265,17 @@ run_test(Witnesses, Config) -> lists:zip(AllGws, GatewayAddrs) ), + %% For crosscheck + GatewayLetterLocMap = lists:foldl( + fun({Letter, A}, Acc) -> + {ok, Gw} = blockchain_ledger_v1:find_gateway_info(A, Ledger), + GwLoc = blockchain_ledger_gateway_v2:location(Gw), + maps:put(Letter, GwLoc, Acc) + end, + #{}, + lists:zip(AllGws, GatewayAddrs) + ), + Challenger = maps:get(k, GatewayLetterToAddrMap), GwA = maps:get(a, GatewayLetterToAddrMap), @@ -422,6 +434,7 @@ run_test(Witnesses, Config) -> %% Theoretically, gateways J, K should have higher witness rewards than B, C, E, F, G, I ct:pal("GatewayNameMap: ~p", [GatewayNameMap]), ct:pal("GatewayLocMap: ~p", [GatewayLocMap]), + ct:pal("GatewayLetterLocMap: ~p", [GatewayLetterLocMap]), ct:pal("ChallengeesRewardsMap: ~p", [ChallengeesRewardsMap]), ct:pal("WitnessRewardsMap: ~p", [WitnessRewardsMap]), @@ -463,3 +476,60 @@ hip17_vars() -> ?hip17_res_12 => <<"2,100000,100000">>, ?density_tgt_res => 8 }. + +known_locations() -> + + %% NameToPubkeyBin = + %% #{ + %% "rare-amethyst-reindeer" => libp2p_crypto:b58_to_bin("112VfPXs1WTRjp24WkbbjQmbFNhb5ptot7gpGwJULm4SRiacjuvW"), + %% "cold-canvas-duck" => libp2p_crypto:b58_to_bin("112ciDxDUBwJZs5YjjPWJWKGwGtUtdJdxSgDAYJPhu9fHw4sgeQy"), + %% "melted-tangelo-aphid" => libp2p_crypto:b58_to_bin("112eNuzPYSeeo3tqNDidr2gPysz7QtLkePkY5Yn1V7ddNPUDN6p5"), + %% "early-lime-rat" => libp2p_crypto:b58_to_bin("112Xr4ZtiNbeh8wfiWTYfeo7KwBbXwvx5F2LPdNTC8wp8q4EQCAm"), + %% "flat-lilac-shrimp" => libp2p_crypto:b58_to_bin("112euXBKmLzUAfyi7FaYRxRpcH5RmfPKprV3qEyHCTt8nqwyVFYo"), + %% "harsh-sandstone-stork" => libp2p_crypto:b58_to_bin("11UFysjjP9W8S7ZV54iK7L6HpkxkHrSPRm4rKkWq22cStYYhDhM"), + %% "pet-pewter-lobster" => libp2p_crypto:b58_to_bin("112LYrRkJX32jVNsuAzt9kDqrddXqWrwpG8N5QX2hELvzf8JJZbw"), + %% "amateur-tan-monkey" => libp2p_crypto:b58_to_bin("112bQKSN3TiaYMrsjNKGZotd14QPi7DB37FeV88rmVMgP4MgTK9q"), + %% "clean-wooden-zebra" => libp2p_crypto:b58_to_bin("112AT5baYcYG6yHchYa9xnkqNJ4cXbgxCh8i8nvnfZeewAHJ8zKc"), + %% "odd-champagne-nuthatch" => libp2p_crypto:b58_to_bin("112fDV4b5FqSgcnu3F592RauuNo5HkuPzfETM7WJ9AfCdaCo9sLk"), + %% "abundant-grape-butterfly" => libp2p_crypto:b58_to_bin("112pdh3waHFbu3XqtCWwbw9xEtYtUEvbqzgSVbEoENBRQznj9Tuy") + %% }, + + %% Locs = + %% #{ + %% "rare-amethyst-reindeer" => 631786582666296319, + %% "cold-canvas-duck" => 631786582410363903, + %% "melted-tangelo-aphid" => 631786582655116287, + %% "early-lime-rat" => 631786582659491327, + %% "flat-lilac-shrimp" => 631786581941828607, + %% "harsh-sandstone-stork" => 631786581946850303, + %% "pet-pewter-lobster" => 631786581906280959, + %% "amateur-tan-monkey" => 631786581937244159, + %% "clean-wooden-zebra" => 631786581846989823, + %% "odd-champagne-nuthatch" => 631786581944091647, + %% "abundant-grape-butterfly" => 631786582694056959 + %% }, + + %% [631786581906280959, + %% 631786582666296319, + %% 631786582410363903, + %% 631786582694056959, + %% 631786582655116287, + %% 631786582659491327, + %% 631786581941828607, + %% 631786581937244159, + %% 631786581946850303, + %% 631786581846989823, + %% 631786581944091647], + + + [631211351866199551, + 631211351866199551, + 631211351866084351, + 631211351866223615, + 631211351866300415, + 631211351866239999, + 631211351866165759, + 631211351866165247, + 631211351866289663, + 631211351865407487, + 631211351865991679]. diff --git a/test/test_utils.erl b/test/test_utils.erl index c521a51e83..3a68476a68 100644 --- a/test/test_utils.erl +++ b/test/test_utils.erl @@ -7,6 +7,7 @@ -export([ init/1, init/2, init_chain/2, init_chain/3, init_chain/4, + init_chain_with_fixed_locations/5, generate_keys/1, generate_keys/2, wait_until/1, wait_until/3, create_block/2, create_block/3, create_block/4, @@ -41,20 +42,23 @@ init_chain(Balance, Keys) -> init_chain(Balance, Keys, true, #{}). -init_chain(Balance, {PrivKey, PubKey}, InConsensus, ExtraVars) -> - % Generate fake blockchains (just the keys) - GenesisMembers = case InConsensus of - true -> - RandomKeys = test_utils:generate_keys(10), - Address = blockchain_swarm:pubkey_bin(), - [ - {Address, {PubKey, PrivKey, libp2p_crypto:mk_sig_fun(PrivKey)}} - ] ++ RandomKeys; - false -> - test_utils:generate_keys(11) - end, +init_chain(Balance, {_PrivKey, _PubKey}=Keys, InConsensus, ExtraVars) -> + GenesisMembers = init_genesis_members(Keys, InConsensus), init_chain(Balance, GenesisMembers, ExtraVars). +init_genesis_members({PrivKey, PubKey}, InConsensus) -> + % Generate fake blockchains (just the keys) + case InConsensus of + true -> + RandomKeys = test_utils:generate_keys(10), + Address = blockchain_swarm:pubkey_bin(), + [ + {Address, {PubKey, PrivKey, libp2p_crypto:mk_sig_fun(PrivKey)}} + ] ++ RandomKeys; + false -> + test_utils:generate_keys(11) + end. + init_chain(Balance, Keys, InConsensus) when is_tuple(Keys), is_boolean(InConsensus) -> init_chain(Balance, Keys, InConsensus, #{}); init_chain(Balance, GenesisMembers, ExtraVars) when is_list(GenesisMembers), is_map(ExtraVars) -> @@ -103,6 +107,48 @@ init_chain(Balance, GenesisMembers, ExtraVars) when is_list(GenesisMembers), is_ ?assertEqual({ok, 1}, blockchain:height(Chain)), {ok, GenesisMembers, GenesisBlock, ConsensusMembers, Keys}. +init_chain_with_fixed_locations(Balance, {PrivKey, PubKey}, InConsensus, Locations, ExtraVars) when is_list(Locations), is_map(ExtraVars) -> + + GenesisMembers = init_genesis_members({PrivKey, PubKey}, InConsensus), + + % Create genesis block + {InitialVars, Keys} = blockchain_ct_utils:create_vars(ExtraVars), + + GenPaymentTxs = [blockchain_txn_coinbase_v1:new(Addr, Balance) + || {Addr, _} <- GenesisMembers], + + GenSecPaymentTxs = [blockchain_txn_security_coinbase_v1:new(Addr, Balance) + || {Addr, _} <- GenesisMembers], + + Addresses = [Addr || {Addr, _} <- GenesisMembers], + + InitialGatewayTxn = [blockchain_txn_gen_gateway_v1:new(Addr, Addr, Loc, 0) + || {Addr, Loc} <- lists:zip(Addresses, Locations)], + + ConsensusMembers = lists:sublist(GenesisMembers, 7), + GenConsensusGroupTx = blockchain_txn_consensus_group_v1:new( + [Addr || {Addr, _} <- ConsensusMembers], <<"proof">>, 1, 0), + Txs = InitialVars ++ + GenPaymentTxs ++ + GenSecPaymentTxs ++ + InitialGatewayTxn ++ + [GenConsensusGroupTx], + lager:info("initial transactions: ~p", [Txs]), + + GenesisBlock = blockchain_block:new_genesis_block(Txs), + ok = blockchain_worker:integrate_genesis_block(GenesisBlock), + + Chain = blockchain_worker:blockchain(), + {ok, HeadBlock} = blockchain:head_block(Chain), + ok = test_utils:wait_until(fun() ->{ok, 1} =:= blockchain:height(Chain) end), + + ?assertEqual(blockchain_block:hash_block(GenesisBlock), blockchain_block:hash_block(HeadBlock)), + ?assertEqual({ok, GenesisBlock}, blockchain:head_block(Chain)), + ?assertEqual({ok, blockchain_block:hash_block(GenesisBlock)}, blockchain:genesis_hash(Chain)), + ?assertEqual({ok, GenesisBlock}, blockchain:genesis_block(Chain)), + ?assertEqual({ok, 1}, blockchain:height(Chain)), + {ok, GenesisMembers, GenesisBlock, ConsensusMembers, Keys}. + generate_keys(N) -> generate_keys(N, ecc_compact). From 0c87807dd59ae82c85c69a19b3476f23293ac99e Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Mon, 7 Dec 2020 14:40:12 -0800 Subject: [PATCH 11/18] More cross verification --- src/blockchain_hex.erl | 27 ++++++- .../v1/blockchain_txn_rewards_v1.erl | 78 ++++++++++++------- test/blockchain_reward_hip17_SUITE.erl | 32 +++++--- 3 files changed, 99 insertions(+), 38 deletions(-) diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl index 16129f9a1d..96a62173f0 100644 --- a/src/blockchain_hex.erl +++ b/src/blockchain_hex.erl @@ -1,6 +1,8 @@ -module(blockchain_hex). --export([var_map/1, scale/4, destroy_memoization/0]). +-export([var_map/1, + scale/2, scale/4, + destroy_memoization/0]). -ifdef(TEST). -export([densities/3]). @@ -31,6 +33,21 @@ destroy_memoization() -> try ets:delete(?SCALE_MEMO_TBL) catch _:_ -> true end, try ets:delete(?DENSITY_MEMO_TBL) catch _:_ -> true end. +%% @doc This call is for blockchain_etl to use directly +-spec scale(Location :: h3:h3_index(), + Ledger :: blockchain_ledger_v1:ledger()) -> {error, any()} | {ok, float()}. +scale(Location, Ledger) -> + case var_map(Ledger) of + {error, _}=E -> E; + {ok, VarMap} -> + case get_target_res(Ledger) of + {error, _}=E -> E; + {ok, TargetRes} -> + S = scale(Location, VarMap, TargetRes, Ledger), + {ok, S} + end + end. + -spec scale( Location :: h3:h3_index(), VarMap :: var_map(), @@ -347,3 +364,11 @@ get_density_var(Var, Ledger) -> ], {ok, [N, Tgt, Max]} end. + +-spec get_target_res(Ledger :: blockchain_ledger_v1:ledger()) -> {error, any()} | {ok, non_neg_integer()}. +get_target_res(Ledger) -> + case blockchain:config(?density_tgt_res, Ledger) of + {error, _}=E -> E; + {ok, V} -> V + end. + diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index 4b21bdb617..0653f2d75d 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -269,19 +269,27 @@ get_rewards_for_epoch(Current, End, Chain, Vars, Ledger, ChallengerRewards, Chal dc_rewards(Transactions, End, Vars, Ledger, DCRewards)); {ok, VarMap} -> + WR = poc_witnesses_rewards(Transactions, Vars, + Chain, Ledger, + WitnessRewards, VarMap), + + C0R = poc_challengers_rewards(Transactions, Vars, + ChallengerRewards), + + DCR = dc_rewards(Transactions, End, Vars, + Ledger, DCRewards), + + CR = poc_challengees_rewards(Transactions, Vars, + Chain, Ledger, + ChallengeeRewards, + VarMap), + %% do the new thing get_rewards_for_epoch(Current+1, End, Chain, Vars, Ledger, - poc_challengers_rewards(Transactions, Vars, - ChallengerRewards), - poc_challengees_rewards(Transactions, Vars, - Chain, Ledger, - ChallengeeRewards, - VarMap), - poc_witnesses_rewards(Transactions, Vars, - Chain, Ledger, - WitnessRewards, VarMap), - dc_rewards(Transactions, End, Vars, - Ledger, DCRewards)) + C0R, + CR, + WR, + DCR) end end end. @@ -619,7 +627,9 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, ChallengeeLoc, VarMap, Ledger), - maps:put(Challengee, I+(ToAdd * TxScale), Acc0) + X = maps:put(Challengee, I+(ToAdd * TxScale), Acc0), + lager:info("MARKER1 +++++++~p", [print_intermediate_reward_map(X)]), + X end; false when is_integer(Version), Version > 4 -> %% this challengee rx'd and sent a receipt @@ -633,7 +643,9 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, ChallengeeLoc, VarMap, Ledger), - maps:put(Challengee, I+(ToAdd * TxScale), Acc0) + X = maps:put(Challengee, I+(ToAdd * TxScale), Acc0), + lager:info("MARKER2 +++++++~p", [print_intermediate_reward_map(X)]), + X end; _ -> case poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) of @@ -646,7 +658,9 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, ChallengeeLoc, VarMap, Ledger), - maps:put(Challengee, I+(ToAdd * TxScale), Acc0) + X = maps:put(Challengee, I+(ToAdd * TxScale), Acc0), + lager:info("MARKER3 +++++++~p", [print_intermediate_reward_map(X)]), + X end end, poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1); @@ -672,7 +686,9 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, ChallengeeLoc, VarMap, Ledger), - maps:put(Challengee, I+(ToAdd * TxScale), Acc0) + X = maps:put(Challengee, I+(ToAdd * TxScale), Acc0), + lager:info("MARKER4 +++++++~p", [print_intermediate_reward_map(X)]), + X end; true -> case poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) of @@ -685,7 +701,9 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, ChallengeeLoc, VarMap, Ledger), - maps:put(Challengee, I+(ToAdd * TxScale), Acc0) + X = maps:put(Challengee, I+(ToAdd * TxScale), Acc0), + lager:info("MARKER5 +++++++~p", [print_intermediate_reward_map(X)]), + X end end, poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1) @@ -750,7 +768,7 @@ poc_witnesses_rewards(Transactions, Path = blockchain_txn_poc_receipts_v1:path(Txn), %% Do the new thing for witness filtering - lists:foldl( + Res = lists:foldl( fun(Elem, Acc1) -> ElemPos = blockchain_utils:index_of(Elem, Path), WitnessChannel = lists:nth(ElemPos, Channels), @@ -777,10 +795,10 @@ poc_witnesses_rewards(Transactions, undefined -> %% old (HIP15) lists:foldl( - fun(WitnessRecord, Map) -> + fun(WitnessRecord, Acc2) -> Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), - I = maps:get(Witness, Map, 0), - maps:put(Witness, I+ToAdd, Map) + I = maps:get(Witness, Acc2, 0), + maps:put(Witness, I+ToAdd, Acc2) end, Acc1, ValidWitnesses @@ -788,13 +806,13 @@ poc_witnesses_rewards(Transactions, D -> %% new (HIP17) lists:foldl( - fun(WitnessRecord, Map) -> + fun(WitnessRecord, Acc2) -> Challengee = blockchain_poc_path_element_v1:challengee(Elem), case blockchain_ledger_v1:find_gateway_info(Challengee, Ledger) of {ok, ChallengeeGw} -> case blockchain_ledger_gateway_v2:location(ChallengeeGw) of undefined -> - Map; + Acc2; ChallengeeLoc -> Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), %% The witnesses get scaled by the value of their transmitters @@ -802,12 +820,13 @@ poc_witnesses_rewards(Transactions, VarMap, D, Ledger), - lager:info("WitnessGw: ~p, RxScale: ~p", [blockchain_utils:addr2name(Witness), RxScale]), - I = maps:get(Witness, Map, 0), - maps:put(Witness, I+(ToAdd*RxScale), Map) + lager:info("WitnessGw: ~p, RxScale: ~p", + [blockchain_utils:addr2name(Witness), RxScale]), + I = maps:get(Witness, Acc2, 0), + maps:put(Witness, I+(ToAdd*RxScale), Acc2) end; _ -> - Map + Acc2 end end, Acc1, @@ -818,7 +837,9 @@ poc_witnesses_rewards(Transactions, end, Acc0, Path - ) + ), + lager:info("Reward Result: ~p", [print_intermediate_reward_map(Res)]), + Res catch What:Why:ST -> lager:error("failed to calculate poc_witnesses_rewards, error ~p:~p:~p", [What, Why, ST]), Acc0 @@ -1104,6 +1125,9 @@ share_of_dc_rewards(_Key, #{dc_remainder := 0}) -> share_of_dc_rewards(Key, Vars=#{dc_remainder := DCRemainder}) -> erlang:round(DCRemainder * ((maps:get(Key, Vars) / (maps:get(poc_challengers_percent, Vars) + maps:get(poc_challengees_percent, Vars) + maps:get(poc_witnesses_percent, Vars))))). +print_intermediate_reward_map(Map) -> + maps:fold(fun(K, Val, Acc) -> maps:put(blockchain_utils:addr2name(K), Val, Acc) end, #{}, Map). + %% ------------------------------------------------------------------ %% EUNIT Tests %% ------------------------------------------------------------------ diff --git a/test/blockchain_reward_hip17_SUITE.erl b/test/blockchain_reward_hip17_SUITE.erl index 604ace25c5..2f72dccedc 100644 --- a/test/blockchain_reward_hip17_SUITE.erl +++ b/test/blockchain_reward_hip17_SUITE.erl @@ -452,7 +452,7 @@ run_test(Witnesses, Config) -> hip15_vars() -> #{ %% configured on chain - ?poc_version => 9, + ?poc_version => 10, ?reward_version => 5, %% new hip15 vars for testing ?poc_reward_decay_rate => 0.8, @@ -522,14 +522,26 @@ known_locations() -> %% 631786581944091647], + %% [631211351866199551, + %% 631211351866199551, + %% 631211351866084351, + %% 631211351866223615, + %% 631211351866300415, + %% 631211351866239999, + %% 631211351866165759, + %% 631211351866165247, + %% 631211351866289663, + %% 631211351865407487, + %% 631211351865991679], + [631211351866199551, 631211351866199551, - 631211351866084351, - 631211351866223615, - 631211351866300415, - 631211351866239999, - 631211351866165759, - 631211351866165247, - 631211351866289663, - 631211351865407487, - 631211351865991679]. + 631211351866199551, + 631211351866199551, + 631211351866199551, + 631211351866199551, + 631211351866199551, + 631211351866199551, + 631211351866199551, + 631211351866199551, + 631211351866199551]. From 046e9cc0d47ad30c68dccd7ff086900d9c253292 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Mon, 7 Dec 2020 22:49:36 -0600 Subject: [PATCH 12/18] Add a marker to BC_UPGRADE_NAMES --- include/blockchain.hrl | 2 +- src/blockchain_hex.erl | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/include/blockchain.hrl b/include/blockchain.hrl index 90f9aecc1b..3a17d7962f 100644 --- a/include/blockchain.hrl +++ b/include/blockchain.hrl @@ -25,4 +25,4 @@ % Misc -define(EVT_MGR, blockchain_event_mgr). --define(BC_UPGRADE_NAMES, [<<"gateway_v2">>, <<"hex_targets">>, <<"gateway_oui">>]). +-define(BC_UPGRADE_NAMES, [<<"gateway_v2">>, <<"hex_targets">>, <<"gateway_oui">>, <<"h3dex">>]). diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl index 96a62173f0..d94886bc64 100644 --- a/src/blockchain_hex.erl +++ b/src/blockchain_hex.erl @@ -20,6 +20,7 @@ -type var_map() :: #{0..12 => map()}. -type locations() :: #{h3:h3_index() => [libp2p_crypto:pubkey_bin(), ...]}. -type h3_indices() :: [h3:h3_index()]. +-type tblnames() :: ?SCALE_MEMO_TBL|?DENSITY_MEMO_TBL. -export_type([var_map/0]). @@ -64,10 +65,6 @@ scale(Location, VarMap, TargetRes, Ledger) -> calculate_scale(Location, VarMap, TargetRes, Ledger)) end. - -%% TODO: This ought to be stored in the ledger because it won't change much? ever? -%% after it's been computed. Seems dumb to calculate it every single time we pay -%% out rewards. -spec var_map(Ledger :: blockchain_ledger_v1:ledger()) -> {error, any()} | {ok, var_map()}. %% @doc This function returns a map of hex resolutions mapped to hotspot density targets and %% maximums. These numbers are used during PoC witness and challenge rewards calculations. @@ -119,7 +116,7 @@ var_map(Ledger) -> %% Internal functions %%-------------------------------------------------------------------- -spec lookup(Key :: term(), - TblName :: ?SCALE_MEMO_TBL|?DENSITY_MEMO_TBL) -> + TblName :: tblnames()) -> {ok, Result :: term()} | not_found. lookup(Key, TblName) -> try @@ -129,19 +126,20 @@ lookup(Key, TblName) -> end catch %% if the table doesn't exist yet, create it and return `not_found' - _:badarg -> + error:badarg -> _ = maybe_start(TblName), not_found end. +-spec maybe_start(TblName :: tblnames()) -> tblnames(). maybe_start(TblName) -> try - _Name = ets:new(TblName, ?ETS_OPTS) + _TblName = ets:new(TblName, ?ETS_OPTS) catch - _:_ -> TblName + error:badarg -> TblName end. --spec memoize(TblName :: ?SCALE_MEMO_TBL|?DENSITY_MEMO_TBL, +-spec memoize(TblName :: tblnames(), Key :: term(), Result :: term()) -> Result :: term(). memoize(TblName, Key, Result) -> From 6ec1270d6828bc1393696f7880e8e75b197be986 Mon Sep 17 00:00:00 2001 From: Evan Vigil-McClanahan Date: Tue, 8 Dec 2020 14:48:10 -0800 Subject: [PATCH 13/18] add v5 snapshots with h3dex --- .../v1/blockchain_ledger_snapshot_v1.erl | 335 ++++++++++++++---- test/blockchain_snapshot_SUITE.erl | 3 + 2 files changed, 268 insertions(+), 70 deletions(-) diff --git a/src/ledger/v1/blockchain_ledger_snapshot_v1.erl b/src/ledger/v1/blockchain_ledger_snapshot_v1.erl index 52104a7219..f7a8fc111f 100644 --- a/src/ledger/v1/blockchain_ledger_snapshot_v1.erl +++ b/src/ledger/v1/blockchain_ledger_snapshot_v1.erl @@ -31,7 +31,8 @@ snapshot(Ledger0, Blocks) -> Parent = self(), Ref = make_ref(), - {_Pid, MonitorRef} = spawn_opt(fun ThisFun() -> + {_Pid, MonitorRef} = + spawn_opt(fun ThisFun() -> Ledger = blockchain_ledger_v1:mode(delayed, Ledger0), {ok, CurrHeight} = blockchain_ledger_v1:current_height(Ledger), %% this should not leak a huge amount of atoms @@ -109,6 +110,7 @@ generate_snapshot(Ledger0, Blocks) -> {ok, OUICounter} = blockchain_ledger_v1:get_oui_counter(Ledger), Hexes = blockchain_ledger_v1:snapshot_hexes(Ledger), + H3dex = blockchain_ledger_v1:snapshot_h3dex(Ledger), StateChannels = blockchain_ledger_v1:snapshot_state_channels(Ledger), @@ -116,45 +118,47 @@ generate_snapshot(Ledger0, Blocks) -> {ok, OraclePriceList} = blockchain_ledger_v1:current_oracle_price_list(Ledger), Snapshot = - #blockchain_snapshot_v4{ - current_height = CurrHeight, - transaction_fee = 0, - consensus_members = ConsensusMembers, + #{ + version => v5, + current_height => CurrHeight, + transaction_fee => 0, + consensus_members => ConsensusMembers, - election_height = ElectionHeight, - election_epoch = ElectionEpoch, + election_height => ElectionHeight, + election_epoch => ElectionEpoch, - delayed_vars = DelayedVars, - threshold_txns = ThresholdTxns, + delayed_vars => DelayedVars, + threshold_txns => ThresholdTxns, - master_key = MasterKey, - multi_keys = MultiKeys, - vars_nonce = VarsNonce, - vars = Vars, + master_key => MasterKey, + multi_keys => MultiKeys, + vars_nonce => VarsNonce, + vars => Vars, - gateways = Gateways, - pocs = PoCs, + gateways => Gateways, + pocs => PoCs, - accounts = Accounts, - dc_accounts = DCAccounts, + accounts => Accounts, + dc_accounts => DCAccounts, - security_accounts = SecurityAccounts, + security_accounts => SecurityAccounts, - htlcs = HTLCs, + htlcs => HTLCs, - ouis = OUIs, - subnets = Subnets, - oui_counter = OUICounter, + ouis => OUIs, + subnets => Subnets, + oui_counter => OUICounter, - hexes = Hexes, + hexes => Hexes, + h3dex => H3dex, - state_channels = StateChannels, + state_channels => StateChannels, - blocks = Blocks, + blocks => Blocks, - oracle_price = OraclePrice, - oracle_price_list = OraclePriceList - }, + oracle_price => OraclePrice, + oracle_price_list => OraclePriceList + }, {ok, Snapshot} catch C:E:S -> @@ -170,9 +174,9 @@ serialize(Snapshot, BlocksP) -> Blocks = lists:map(fun(B) when is_tuple(B) -> blockchain_block:serialize(B); (B) -> B - end, Snapshot#blockchain_snapshot_v4.blocks), - Snapshot#blockchain_snapshot_v4{blocks = Blocks}; - noblocks -> Snapshot#blockchain_snapshot_v4{blocks = []} + end, maps:get(blocks, Snapshot, [])), + lists:sort(maps:to_list(Snapshot#{blocks => Blocks})); + noblocks -> Snapshot#{blocks => []} end, Bin = term_to_binary(Snapshot1, [{compressed, 9}]), BinSz = byte_size(Bin), @@ -215,12 +219,24 @@ serialize_v3(Snapshot, noblocks) -> BinSz:32/little-unsigned-integer, Bin/binary>>, {ok, Snap}. +serialize_v4(Snapshot, noblocks) -> + %% NOTE: serialize_v4 only gets called with noblocks + Snapshot1 = Snapshot#blockchain_snapshot_v4{blocks = []}, + Bin = term_to_binary(Snapshot1, [{compressed, 9}]), + BinSz = byte_size(Bin), + + %% do some simple framing with version, size, & snap + Snap = <<4, %% version + BinSz:32/little-unsigned-integer, Bin/binary>>, + {ok, Snap}. + + deserialize(<<1, %%SHASz:16/little-unsigned-integer, SHA:SHASz/binary, BinSz:32/little-unsigned-integer, BinSnap:BinSz/binary>>) -> try binary_to_term(BinSnap) of OldSnapshot -> - Snapshot = v3_to_v4(v2_to_v3(v1_to_v2(OldSnapshot))), + Snapshot = v4_to_v5(v3_to_v4(v2_to_v3(v1_to_v2(OldSnapshot)))), {ok, Snapshot} catch _:_ -> {error, bad_snapshot_binary} @@ -230,7 +246,7 @@ deserialize(<<2, BinSz:32/little-unsigned-integer, BinSnap:BinSz/binary>>) -> try binary_to_term(BinSnap) of OldSnapshot -> - Snapshot = v3_to_v4(v2_to_v3(OldSnapshot)), + Snapshot = v4_to_v5(v3_to_v4(v2_to_v3(OldSnapshot))), {ok, Snapshot} catch _:_ -> {error, bad_snapshot_binary} @@ -240,7 +256,7 @@ deserialize(<<3, BinSz:32/little-unsigned-integer, BinSnap:BinSz/binary>>) -> try binary_to_term(BinSnap) of OldSnapshot -> - Snapshot = v3_to_v4(OldSnapshot), + Snapshot = v4_to_v5(v3_to_v4(OldSnapshot)), {ok, Snapshot} catch _:_ -> {error, bad_snapshot_binary} @@ -249,7 +265,17 @@ deserialize(<<4, %%SHASz:16/little-unsigned-integer, SHA:SHASz/binary, BinSz:32/little-unsigned-integer, BinSnap:BinSz/binary>>) -> try binary_to_term(BinSnap) of - #blockchain_snapshot_v4{} = Snapshot -> + OldSnapshot -> + Snapshot = v4_to_v5(OldSnapshot), + {ok, Snapshot} + catch _:_ -> + {error, bad_snapshot_binary} + end; +deserialize(<<5, + %%SHASz:16/little-unsigned-integer, SHA:SHASz/binary, + BinSz:32/little-unsigned-integer, BinSnap:BinSz/binary>>) -> + try maps:from_list(binary_to_term(BinSnap)) of + #{version := v5} = Snapshot -> {ok, Snapshot} catch _:_ -> {error, bad_snapshot_binary} @@ -257,50 +283,51 @@ deserialize(<<4, %% sha will be stored externally import(Chain, SHA, - #blockchain_snapshot_v4{ - current_height = CurrHeight, - transaction_fee = _TxnFee, - consensus_members = ConsensusMembers, + #{ + current_height := CurrHeight, + transaction_fee := _TxnFee, + consensus_members := ConsensusMembers, - election_height = ElectionHeight, - election_epoch = ElectionEpoch, + election_height := ElectionHeight, + election_epoch := ElectionEpoch, - delayed_vars = DelayedVars, - threshold_txns = ThresholdTxns, + delayed_vars := DelayedVars, + threshold_txns := ThresholdTxns, - master_key = MasterKey, - multi_keys = MultiKeys, - vars_nonce = VarsNonce, - vars = Vars, + master_key := MasterKey, + multi_keys := MultiKeys, + vars_nonce := VarsNonce, + vars := Vars, - gateways = Gateways, - pocs = PoCs, + gateways := Gateways, + pocs := PoCs, - accounts = Accounts, - dc_accounts = DCAccounts, + accounts := Accounts, + dc_accounts := DCAccounts, - security_accounts = SecurityAccounts, + security_accounts := SecurityAccounts, - htlcs = HTLCs, + htlcs := HTLCs, - ouis = OUIs, - subnets = Subnets, - oui_counter = OUICounter, + ouis := OUIs, + subnets := Subnets, + oui_counter := OUICounter, - hexes = Hexes, + hexes := Hexes, - state_channels = StateChannels, + state_channels := StateChannels, - blocks = Blocks, + blocks := Blocks, - oracle_price = OraclePrice, - oracle_price_list = OraclePriceList + oracle_price := OraclePrice, + oracle_price_list := OraclePriceList } = Snapshot) -> Dir = blockchain:dir(Chain), case hash(Snapshot) == SHA orelse - hash_v3(v4_to_v3(Snapshot)) == SHA orelse - hash_v2(v3_to_v2(v4_to_v3(Snapshot))) == SHA orelse - hash_v1(v2_to_v1(v3_to_v2(v4_to_v3(Snapshot)))) == SHA of + hash_v4(v5_to_v4(Snapshot)) == SHA orelse + hash_v3(v4_to_v3(v5_to_v4(Snapshot))) == SHA orelse + hash_v2(v3_to_v2(v4_to_v3(v5_to_v4(Snapshot)))) == SHA orelse + hash_v1(v2_to_v1(v3_to_v2(v4_to_v3(v5_to_v4(Snapshot))))) == SHA of true -> CLedger = blockchain:ledger(Chain), Ledger0 = @@ -439,10 +466,10 @@ get_blocks(Chain) -> end || N <- lists:seq(max(?min_height, LoadBlockStart), Height)]. -height(#blockchain_snapshot_v4{current_height = Height}) -> +height(#{current_height := Height}) -> Height. -hash(#blockchain_snapshot_v4{} = Snap) -> +hash(#{version := v5} = Snap) -> {ok, BinSnap} = serialize(Snap, noblocks), crypto:hash(sha256, BinSnap). @@ -458,6 +485,10 @@ hash_v3(#blockchain_snapshot_v3{} = Snap) -> {ok, BinSnap} = serialize_v3(Snap, noblocks), crypto:hash(sha256, BinSnap). +hash_v4(#blockchain_snapshot_v4{} = Snap) -> + {ok, BinSnap} = serialize_v4(Snap, noblocks), + crypto:hash(sha256, BinSnap). + v1_to_v2(#blockchain_snapshot_v1{ previous_snapshot_hash = <<>>, leading_hash = <<>>, @@ -784,6 +815,87 @@ v3_to_v4(#blockchain_snapshot_v3{ oracle_price_list = OraclePriceList }. +v4_to_v5(#blockchain_snapshot_v4{ + current_height = CurrHeight, + transaction_fee = _TxnFee, + consensus_members = ConsensusMembers, + + election_height = ElectionHeight, + election_epoch = ElectionEpoch, + + delayed_vars = DelayedVars, + threshold_txns = ThresholdTxns, + + master_key = MasterKey, + multi_keys = MultiKeys, + vars_nonce = VarsNonce, + vars = Vars, + + gateways = Gateways, + pocs = PoCs, + + accounts = Accounts, + dc_accounts = DCAccounts, + + security_accounts = SecurityAccounts, + + htlcs = HTLCs, + + ouis = OUIs, + subnets = Subnets, + oui_counter = OUICounter, + + hexes = Hexes, + + state_channels = StateChannels, + + blocks = Blocks, + + oracle_price = OraclePrice, + oracle_price_list = OraclePriceList + }) -> + #{ + version => v5, + current_height => CurrHeight, + transaction_fee => 0, + consensus_members => ConsensusMembers, + + election_height => ElectionHeight, + election_epoch => ElectionEpoch, + + delayed_vars => DelayedVars, + threshold_txns => ThresholdTxns, + + master_key => MasterKey, + multi_keys => MultiKeys, + vars_nonce => VarsNonce, + vars => Vars, + + gateways => Gateways, + pocs => PoCs, + + accounts => Accounts, + dc_accounts => DCAccounts, + + security_accounts => SecurityAccounts, + + htlcs => HTLCs, + + ouis => OUIs, + subnets => Subnets, + oui_counter => OUICounter, + + hexes => Hexes, + h3dex => [], + + state_channels => StateChannels, + + blocks => Blocks, + + oracle_price => OraclePrice, + oracle_price_list => OraclePriceList + }. + reserialize(Fun, Values) -> lists:map(fun({K, V}) -> @@ -962,6 +1074,84 @@ v4_to_v3(#blockchain_snapshot_v4{ oracle_price_list = OraclePriceList }. +v5_to_v4(#{ + version := v5, + current_height := CurrHeight, + transaction_fee := 0, + consensus_members := ConsensusMembers, + + election_height := ElectionHeight, + election_epoch := ElectionEpoch, + + delayed_vars := DelayedVars, + threshold_txns := ThresholdTxns, + + master_key := MasterKey, + multi_keys := MultiKeys, + vars_nonce := VarsNonce, + vars := Vars, + + gateways := Gateways, + pocs := PoCs, + + accounts := Accounts, + dc_accounts := DCAccounts, + + security_accounts := SecurityAccounts, + + htlcs := HTLCs, + + ouis := OUIs, + subnets := Subnets, + oui_counter := OUICounter, + + hexes := Hexes, + h3dex := _H3dex, + + state_channels := StateChannels, + + blocks := Blocks, + + oracle_price := OraclePrice, + oracle_price_list := OraclePriceList}) -> + #blockchain_snapshot_v4{ + current_height = CurrHeight, + consensus_members = ConsensusMembers, + + election_height = ElectionHeight, + election_epoch = ElectionEpoch, + + delayed_vars = DelayedVars, + threshold_txns = ThresholdTxns, + + master_key = MasterKey, + multi_keys = MultiKeys, + vars_nonce = VarsNonce, + vars = Vars, + + gateways = Gateways, + pocs = PoCs, + + accounts = Accounts, + dc_accounts = DCAccounts, + + security_accounts = SecurityAccounts, + + htlcs = HTLCs, + + ouis = OUIs, + subnets = Subnets, + oui_counter = OUICounter, + + hexes = Hexes, + + state_channels = StateChannels, + + blocks = Blocks, + + oracle_price = OraclePrice, + oracle_price_list = OraclePriceList + }. deserialize(Fun, Values) -> lists:map(fun({K, V}) -> @@ -978,10 +1168,15 @@ deserialize_pocs(Values) -> Values). diff(A, B) -> - Fields = record_info(fields, blockchain_snapshot_v4), - [_ | AL] = tuple_to_list(A), - [_ | BL] = tuple_to_list(B), - Comp = lists:zip3(Fields, AL, BL), + Fields = lists:sort(maps:keys(A)), + Comp = lists:foldl( + fun(Field, Acc) -> + AV = maps:get(Field, A), + BV = maps:get(Field, B), + [{Field, AV, BV} | Acc] + end, + [], + Fields), lists:foldl( fun({Field, AI, BI}, Acc) -> case AI == BI of diff --git a/test/blockchain_snapshot_SUITE.erl b/test/blockchain_snapshot_SUITE.erl index 7186a77f8c..61535e4620 100644 --- a/test/blockchain_snapshot_SUITE.erl +++ b/test/blockchain_snapshot_SUITE.erl @@ -71,6 +71,9 @@ basic_test(_Config) -> {ok, Snapshot1} = blockchain_ledger_snapshot_v1:snapshot(Ledger1, []), ?assertEqual([], blockchain_ledger_snapshot_v1:diff(Snapshot, Snapshot1)), + + ?assertEqual(blockchain_ledger_snapshot_v1:hash(Snapshot), + blockchain_ledger_snapshot_v1:hash(Snapshot1)), ok. From 21c608858a5a9e297fd641cbb2e0d437df24c460 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Mon, 7 Dec 2020 17:23:23 -0800 Subject: [PATCH 14/18] Enhance tests for hip17 rewards --- src/blockchain_hex.erl | 2 +- .../v1/blockchain_txn_rewards_v1.erl | 47 +- test/blockchain_hex_SUITE.erl | 4 +- test/blockchain_reward_hip17_SUITE.erl | 944 +++++++++++++++--- test/test_utils.erl | 22 +- 5 files changed, 813 insertions(+), 206 deletions(-) diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl index d94886bc64..afd141410a 100644 --- a/src/blockchain_hex.erl +++ b/src/blockchain_hex.erl @@ -367,6 +367,6 @@ get_density_var(Var, Ledger) -> get_target_res(Ledger) -> case blockchain:config(?density_tgt_res, Ledger) of {error, _}=E -> E; - {ok, V} -> V + {ok, V} -> {ok, V} end. diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index 0653f2d75d..d9b6440d88 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -238,10 +238,13 @@ get_rewards_for_epoch(Start, End, Chain, Vars, Ledger, DCRewards) -> get_rewards_for_epoch(Start, End, _Chain, Vars0, _Ledger, ChallengerRewards, ChallengeeRewards, WitnessRewards, DCRewards) when Start == End+1 -> {DCRemainder, NewDCRewards} = normalize_dc_rewards(DCRewards, Vars0), Vars = maps:put(dc_remainder, DCRemainder, Vars0), + + NormalizedWitnessRewards = normalize_witness_rewards(WitnessRewards, Vars), + %% apply the DC remainder, if any to the other PoC categories pro rata {ok, normalize_challenger_rewards(ChallengerRewards, Vars), normalize_challengee_rewards(ChallengeeRewards, Vars), - normalize_witness_rewards(WitnessRewards, Vars), + NormalizedWitnessRewards, NewDCRewards}; get_rewards_for_epoch(Current, End, Chain, Vars, Ledger, ChallengerRewards, ChallengeeRewards, WitnessRewards, DCRewards) -> case blockchain:get_block(Current, Chain) of @@ -526,7 +529,6 @@ normalize_challengee_rewards(ChallengeeRewards, #{epoch_reward := EpochReward, TotalChallenged = lists:sum(maps:values(ChallengeeRewards)), ShareOfDCRemainder = share_of_dc_rewards(poc_challengees_percent, Vars), ChallengeesReward = (EpochReward * PocChallengeesPercent) + ShareOfDCRemainder, - lager:info("TotalChallenged: ~p", [TotalChallenged]), maps:fold( fun(Challengee, Challenged, Acc) -> PercentofReward = (Challenged*100/TotalChallenged)/100, @@ -627,9 +629,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, ChallengeeLoc, VarMap, Ledger), - X = maps:put(Challengee, I+(ToAdd * TxScale), Acc0), - lager:info("MARKER1 +++++++~p", [print_intermediate_reward_map(X)]), - X + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; false when is_integer(Version), Version > 4 -> %% this challengee rx'd and sent a receipt @@ -643,9 +643,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, ChallengeeLoc, VarMap, Ledger), - X = maps:put(Challengee, I+(ToAdd * TxScale), Acc0), - lager:info("MARKER2 +++++++~p", [print_intermediate_reward_map(X)]), - X + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; _ -> case poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) of @@ -658,9 +656,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, ChallengeeLoc, VarMap, Ledger), - X = maps:put(Challengee, I+(ToAdd * TxScale), Acc0), - lager:info("MARKER3 +++++++~p", [print_intermediate_reward_map(X)]), - X + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end end, poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1); @@ -686,9 +682,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, ChallengeeLoc, VarMap, Ledger), - X = maps:put(Challengee, I+(ToAdd * TxScale), Acc0), - lager:info("MARKER4 +++++++~p", [print_intermediate_reward_map(X)]), - X + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; true -> case poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) of @@ -701,9 +695,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, ChallengeeLoc, VarMap, Ledger), - X = maps:put(Challengee, I+(ToAdd * TxScale), Acc0), - lager:info("MARKER5 +++++++~p", [print_intermediate_reward_map(X)]), - X + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end end, poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1) @@ -730,7 +722,6 @@ poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) -> {N, R} -> W = length(Witnesses), Unit = poc_reward_tx_unit(R, W, N), - lager:info("poc_challengee_reward_unit: ~p", [Unit]), {ok, normalize_reward_unit(Unit)} end. @@ -787,7 +778,6 @@ poc_witnesses_rewards(Transactions, {N, R} -> W = length(ValidWitnesses), U = poc_witness_reward_unit(R, W, N), - lager:info("poc_witness_reward_unit: ~p", [U]), U end, @@ -820,8 +810,6 @@ poc_witnesses_rewards(Transactions, VarMap, D, Ledger), - lager:info("WitnessGw: ~p, RxScale: ~p", - [blockchain_utils:addr2name(Witness), RxScale]), I = maps:get(Witness, Acc2, 0), maps:put(Witness, I+(ToAdd*RxScale), Acc2) end; @@ -838,7 +826,6 @@ poc_witnesses_rewards(Transactions, Acc0, Path ), - lager:info("Reward Result: ~p", [print_intermediate_reward_map(Res)]), Res catch What:Why:ST -> lager:error("failed to calculate poc_witnesses_rewards, error ~p:~p:~p", [What, Why, ST]), @@ -1051,12 +1038,10 @@ normalize_dc_rewards(DCRewards0, #{epoch_reward := EpochReward, -spec poc_reward_tx_unit(R :: float(), W :: pos_integer(), N :: pos_integer()) -> float(). -poc_reward_tx_unit(R, W, N) when W =< N -> - lager:info("nonorm, R: ~p, W: ~p, N: ~p, poc_reward_tx_unit: ~p", [R, W, N, W / N]), +poc_reward_tx_unit(_R, W, N) when W =< N -> blockchain_utils:normalize_float(W / N); poc_reward_tx_unit(R, W, N) -> NoNorm = 1 + (1 - math:pow(R, (W - N))), - lager:info("nonorm, R: ~p, W: ~p, N: ~p, poc_reward_tx_unit: ~p", [R, W, N, NoNorm]), blockchain_utils:normalize_float(NoNorm). -spec poc_witness_reward_unit(R :: float(), @@ -1076,8 +1061,8 @@ legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version) -> ElemPos = blockchain_utils:index_of(Elem, StaticPath), WitnessChannel = lists:nth(ElemPos, Channels), ValidWitnesses = blockchain_txn_poc_receipts_v1:valid_witnesses(Elem, WitnessChannel, Ledger), - lager:debug("ValidWitnesses: ~p", - [[blockchain_utils:addr2name(blockchain_poc_witness_v1:gateway(W)) || W <- ValidWitnesses]]), + %% lager:info("ValidWitnesses: ~p", + %% [[blockchain_utils:addr2name(blockchain_poc_witness_v1:gateway(W)) || W <- ValidWitnesses]]), ValidWitnesses catch What:Why:ST -> lager:error("failed to calculate poc_challengees_rewards, error ~p:~p:~p", [What, Why, ST]), @@ -1089,7 +1074,7 @@ legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version) -> blockchain_poc_path_element_v1:witnesses(Elem) end. -maybe_calc_tx_scale(Challengee, +maybe_calc_tx_scale(_Challengee, DensityTgtRes, ChallengeeLoc, VarMap, @@ -1099,8 +1084,8 @@ maybe_calc_tx_scale(Challengee, {_, undefined} -> 1.0; {D, Loc} -> TxScale = blockchain_hex:scale(Loc, VarMap, D, Ledger), - lager:info("Challengee: ~p, TxScale: ~p", - [blockchain_utils:addr2name(Challengee), TxScale]), + %% lager:info("Challengee: ~p, TxScale: ~p", + %% [blockchain_utils:addr2name(Challengee), TxScale]), TxScale end. @@ -1125,8 +1110,6 @@ share_of_dc_rewards(_Key, #{dc_remainder := 0}) -> share_of_dc_rewards(Key, Vars=#{dc_remainder := DCRemainder}) -> erlang:round(DCRemainder * ((maps:get(Key, Vars) / (maps:get(poc_challengers_percent, Vars) + maps:get(poc_challengees_percent, Vars) + maps:get(poc_witnesses_percent, Vars))))). -print_intermediate_reward_map(Map) -> - maps:fold(fun(K, Val, Acc) -> maps:put(blockchain_utils:addr2name(K), Val, Acc) end, #{}, Map). %% ------------------------------------------------------------------ %% EUNIT Tests diff --git a/test/blockchain_hex_SUITE.erl b/test/blockchain_hex_SUITE.erl index 57fa5fc81c..68cddb6de8 100644 --- a/test/blockchain_hex_SUITE.erl +++ b/test/blockchain_hex_SUITE.erl @@ -372,7 +372,7 @@ export_scale_data(Ledger, VarMap, DensityTargetResolutions, GatewaysWithLocs) -> ). export_gps_file(Fname, Scales) -> - Header = ["name,latitude,longitude,color,desc"], + Header = ["name,latitude,longitude,h3,color,desc"], Data = lists:foldl( fun({Name, H3, ScaleVal}, Acc) -> @@ -384,6 +384,8 @@ export_gps_file(Fname, Scales) -> "," ++ io_lib:format("~.20f", [Long]) ++ "," ++ + integer_to_list(H3) ++ + "," ++ color(ScaleVal) ++ "," ++ io_lib:format("scale: ~p", [ScaleVal]), diff --git a/test/blockchain_reward_hip17_SUITE.erl b/test/blockchain_reward_hip17_SUITE.erl index 2f72dccedc..ea3ba2b85c 100644 --- a/test/blockchain_reward_hip17_SUITE.erl +++ b/test/blockchain_reward_hip17_SUITE.erl @@ -1,3 +1,46 @@ +%% +%% This is primarily a static test +%% +%% For reference: +%% -------------------------------------------------------------------------------------------------- +%% fake_name real_name letter location scale +%% -------------------------------------------------------------------------------------------------- +%% tall-azure-whale striped-umber-copperhead e 631210990515645439 0.25 +%% basic-seaweed-walrun fresh-honey-lizard b 631210990515609087 0.25 +%% brief-saffron-cricket round-fiery-platypus k 631210990516667903 0.5 +%% mini-eggshell-panther quick-admiral-gecko j 631210990528935935 0.2 +%% silly-azure-chicken curly-ivory-alligator d 631210990528385535 0.2 +%% daring-watermelon-sloth skinny-amber-condor c 631210990528546815 0.2 +%% keen-ultraviolet-guppy fantastic-blue-lemur i 631210990529462783 0.2 +%% sharp-green-swift crazy-silver-platypus a 631210990529337343 0.2 +%% bitter-saffron-ferret icy-paisley-lizard g 631210990524024831 0.33 +%% creamy-porcelain-poodled able-mocha-rhino h 631210990524753919 0.33 +%% blurry-raising-wolf puny-bubblegum-shell f 631210990525267455 0.33 +%% +%% Essentially we will check: +%% +%% - test_between_vars: +%% - Do the rewards follow: no_var > hip15_var >= hip17_var. +%% +%% - test_between_challengees: +%% - Paths tested: [e, h, j], [k, h, j], [g, h, j] +%% +%% - For path: [e, h, j] +%% - e: 0.25, h: 0.33, ignore j +%% - e's witnesses [a, b, c, f] should get scaled using 0.25 +%% - h's witnesses [i] should get scaled using 0.33 +%% +%% - For path: [k, h, j] +%% - k: 0.5, h: 0.33, ignore j +%% - k's witnesses [a, b, c, f] should get scaled using 0.5 +%% - h's witnesses [i] should get scaled using 0.33 +%% +%% - For path: [g, h, j] +%% - g: 0.33, h: 0.33, ignore j +%% - g's witnesses [a, b, c, f] should get scaled using 0.33 +%% - h's witnesses [i] should get scaled using 0.33 +%% + -module(blockchain_reward_hip17_SUITE). -include_lib("common_test/include/ct.hrl"). @@ -7,6 +50,9 @@ -export([ all/0, + groups/0, + init_per_group/2, + end_per_group/2, init_per_suite/1, end_per_suite/1, init_per_testcase/2, @@ -15,33 +61,76 @@ -export([ no_var_test/1, - with_hip15_vars_test/1, - with_hip17_vars_test/1, - comparison_test/1 + only_hip15_vars_test/1, + hip15_17_vars_test/1, + comparison_test/1, + path_ehj/1, + path_khj/1, + path_ghj/1, + compare_challengee_test/1 ]). -all() -> +all_var_tests() -> [ no_var_test, - with_hip15_vars_test, - with_hip17_vars_test, + only_hip15_vars_test, + hip15_17_vars_test, comparison_test ]. +all_challengee_tests() -> + [ + path_ehj, + path_khj, + path_ghj, + compare_challengee_test + ]. + +all() -> + [{group, test_between_vars}, {group, test_between_challengees}]. + +groups() -> + [{test_between_vars, + [], + all_var_tests() + }, + {test_between_challengees, + [], + all_challengee_tests() + }]. + +%%-------------------------------------------------------------------- +%% TEST GROUP SETUP +%%-------------------------------------------------------------------- +init_per_group(_, Config) -> + {ok, StorePid} = blockchain_test_reward_store:start(), + [{store, StorePid} | Config]. + +%%-------------------------------------------------------------------- +%% TEST GROUP TEARDOWN +%%-------------------------------------------------------------------- +end_per_group(_, _Config) -> + ok = blockchain_test_reward_store:stop(), + ok. + %%-------------------------------------------------------------------- %% TEST SUITE SETUP %%-------------------------------------------------------------------- init_per_suite(Config) -> - {ok, StorePid} = blockchain_test_reward_store:start(), - [{store, StorePid} | Config]. + + GenesisKeys = static_keys(), + GenesisMembers = [{PubkeyBin,{Pubkey, PrivKey, libp2p_crypto:mk_sig_fun(PrivKey)}} || {PubkeyBin, Pubkey, PrivKey} <- GenesisKeys], + Locations = known_locations(), + + [{genesis_members, GenesisMembers}, + {locations, Locations} | Config]. %%-------------------------------------------------------------------- %% TEST SUITE TEARDOWN %%-------------------------------------------------------------------- end_per_suite(_Config) -> - blockchain_test_reward_store:stop(), ok. %%-------------------------------------------------------------------- @@ -53,20 +142,31 @@ init_per_testcase(TestCase, Config) -> Balance = 5000, {ok, Sup, {PrivKey, PubKey}, Opts} = test_utils:init(?config(base_dir, Config0)), ok = application:set_env(blockchain, hip17_test_mode, true), + application:ensure_all_started(lager), + + BothHip15And17Vars = maps:merge(hip15_vars(), hip17_vars()), ExtraVars = case TestCase of - with_hip15_vars_test -> + only_hip15_vars_test -> hip15_vars(); - with_hip17_vars_test -> - maps:merge(hip15_vars(), hip17_vars()); + hip15_17_vars_test -> + BothHip15And17Vars; + path_ehj -> + BothHip15And17Vars; + path_ghj -> + BothHip15And17Vars; + path_khj -> + BothHip15And17Vars; _ -> #{} end, + GenesisMembers = ?config(genesis_members, Config), + Locations = ?config(locations, Config), - {ok, GenesisMembers, _GenesisBlock, ConsensusMembers, Keys} = - test_utils:init_chain_with_fixed_locations(Balance, {PrivKey, PubKey}, true, known_locations(), ExtraVars), + {ok, _GenesisBlock, ConsensusMembers, Keys} = + test_utils:init_chain_with_fixed_locations(Balance, GenesisMembers, Locations, ExtraVars), Chain = blockchain_worker:blockchain(), Swarm = blockchain_swarm:swarm(), @@ -93,7 +193,18 @@ init_per_testcase(TestCase, Config) -> meck:new(blockchain_txn_rewards_v1, [passthrough]), meck:new(blockchain_txn_poc_receipts_v1, [passthrough]), - meck:new(blockchain_hex, [passthrough]), + meck:new(blockchain_txn_poc_request_v1, [passthrough]), + + case TestCase of + hip15_17_vars_test -> + lists:foreach(fun(Loc) -> + Scale = blockchain_hex:scale(Loc, Ledger), + ct:pal("Loc: ~p, Scale: ~p", [Loc, Scale]) + end, + known_locations()); + _ -> + ok + end, [ {balance, Balance}, @@ -105,7 +216,6 @@ init_per_testcase(TestCase, Config) -> {swarm, Swarm}, {n, N}, {consensus_members, ConsensusMembers}, - {genesis_members, GenesisMembers}, {tc_name, TestCase}, Keys | Config0 @@ -117,10 +227,11 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(_TestCase, Config) -> meck:unload(blockchain_txn_rewards_v1), + meck:unload(blockchain_txn_poc_request_v1), meck:unload(blockchain_txn_poc_receipts_v1), - meck:unload(blockchain_hex), meck:unload(), Sup = ?config(sup, Config), + BaseDir = ?config(base_dir, Config), % Make sure blockchain saved on file = in memory case erlang:is_process_alive(Sup) of true -> @@ -129,6 +240,7 @@ end_per_testcase(_TestCase, Config) -> false -> ok end, + test_utils:cleanup_tmp_dir(BaseDir), ok. %%-------------------------------------------------------------------- @@ -137,9 +249,9 @@ end_per_testcase(_TestCase, Config) -> no_var_test(Config) -> Witnesses = [b, c, e, f, g], - run_test(Witnesses, Config). + run_vars_test(Witnesses, Config). -with_hip15_vars_test(Config) -> +only_hip15_vars_test(Config) -> %% - We have gateways: [a, b, c, d, e, f, g, h, i, j, k] %% - We'll make a poc receipt txn by hand, without any validation %% - We'll also consider that all witnesses are legit (legit_witnesses) @@ -147,9 +259,9 @@ with_hip15_vars_test(Config) -> %% - For a -> d; [b, c, e, f, g] will all be witnesses, their rewards should get scaled %% - For d -> h; 0 witnesses Witnesses = [b, c, e, f, g], - run_test(Witnesses, Config). + run_vars_test(Witnesses, Config). -with_hip17_vars_test(Config) -> +hip15_17_vars_test(Config) -> %% - We have gateways: [a, b, c, d, e, f, g, h, i, j, k] %% - We'll make a poc receipt txn by hand, without any validation %% - We'll also consider that all witnesses are legit (legit_witnesses) @@ -157,7 +269,7 @@ with_hip17_vars_test(Config) -> %% - For a -> d; [b, c, e, f, g] will all be witnesses, their rewards should get scaled %% - For d -> h; 0 witnesses Witnesses = [b, c, e, f, g], - run_test(Witnesses, Config). + run_vars_test(Witnesses, Config). comparison_test(_Config) -> %% Aggregate rewards from no_var_test @@ -171,8 +283,8 @@ comparison_test(_Config) -> FractionalNoVarWitnessRewards = maps:map(fun(_, V) -> V / TotalNoVarWitnessRewards end, NoVarWitnessRewards), %% Aggregate rewards from hip15 test - Hip15ChallengeeRewards = blockchain_test_reward_store:fetch(with_hip15_vars_test_challengee_rewards), - Hip15WitnessRewards = blockchain_test_reward_store:fetch(with_hip15_vars_test_witness_rewards), + Hip15ChallengeeRewards = blockchain_test_reward_store:fetch(only_hip15_vars_test_challengee_rewards), + Hip15WitnessRewards = blockchain_test_reward_store:fetch(only_hip15_vars_test_witness_rewards), TotalHip15ChallengeeRewards = lists:sum(maps:values(Hip15ChallengeeRewards)), TotalHip15WitnessRewards = lists:sum(maps:values(Hip15WitnessRewards)), @@ -181,8 +293,8 @@ comparison_test(_Config) -> FractionalHip15WitnessRewards = maps:map(fun(_, V) -> V / TotalHip15WitnessRewards end, Hip15WitnessRewards), %% Aggregate rewards from hip17 test - Hip17ChallengeeRewards = blockchain_test_reward_store:fetch(with_hip17_vars_test_challengee_rewards), - Hip17WitnessRewards = blockchain_test_reward_store:fetch(with_hip17_vars_test_witness_rewards), + Hip17ChallengeeRewards = blockchain_test_reward_store:fetch(hip15_17_vars_test_challengee_rewards), + Hip17WitnessRewards = blockchain_test_reward_store:fetch(hip15_17_vars_test_witness_rewards), TotalHip17ChallengeeRewards = lists:sum(maps:values(Hip17ChallengeeRewards)), TotalHip17WitnessRewards = lists:sum(maps:values(Hip17WitnessRewards)), @@ -208,106 +320,356 @@ comparison_test(_Config) -> ct:pal("Hip17WitnessRewards: ~p", [Hip17WitnessRewards]), ct:pal("FractionalHip17WitnessRewards: ~p", [FractionalHip17WitnessRewards]), - %% TODO: Actually assert some invariant and compare the three results + + %% challengee rewards for no_vars >= only_hip15_vars >= hip15_17_vars + Challengees = maps:keys(NoVarChallengeeRewards), + + true = lists:all(fun(Challengee) -> + NoVarReward = maps:get(Challengee, NoVarChallengeeRewards), + Hip15Reward = maps:get(Challengee, Hip15ChallengeeRewards), + Hip17Reward = maps:get(Challengee, Hip17ChallengeeRewards), + (NoVarReward >= Hip15Reward) andalso (Hip15Reward >= Hip17Reward) + end, Challengees), + + ok. + + +path_ehj(Config) -> + run_challengees_test(d, e, h, j, [a, b, c, f], [i], Config). + +path_khj(Config) -> + run_challengees_test(d, k, h, j, [a, b, c, f], [i], Config). + +path_ghj(Config) -> + run_challengees_test(d, g, h, j, [a, b, c, f], [i], Config). + +compare_challengee_test(_Config) -> + ChallengeeE = maps:get(e, blockchain_test_reward_store:fetch(path_ehj_challengee_rewards)), + ChallengeeG = maps:get(g, blockchain_test_reward_store:fetch(path_ghj_challengee_rewards)), + ChallengeeK = maps:get(k, blockchain_test_reward_store:fetch(path_khj_challengee_rewards)), + + ct:pal("ChallengeeE: ~p, ChallengeeG: ~p, ChallengeeK: ~p", + [ChallengeeE, ChallengeeG, ChallengeeK]), + + ChallengeeChecks = ((ChallengeeK > ChallengeeG) andalso (ChallengeeK > ChallengeeE) andalso (ChallengeeG > ChallengeeE)), + + true = ChallengeeChecks, + + %% for path e, h, j + %% i gets higher scaled reward + WitnessA_PathEHJ = maps:get(a, blockchain_test_reward_store:fetch(path_ehj_witness_rewards)), + WitnessB_PathEHJ = maps:get(b, blockchain_test_reward_store:fetch(path_ehj_witness_rewards)), + WitnessC_PathEHJ = maps:get(c, blockchain_test_reward_store:fetch(path_ehj_witness_rewards)), + WitnessF_PathEHJ = maps:get(f, blockchain_test_reward_store:fetch(path_ehj_witness_rewards)), + WitnessI_PathEHJ = maps:get(i, blockchain_test_reward_store:fetch(path_ehj_witness_rewards)), + + true = (WitnessA_PathEHJ == WitnessB_PathEHJ) andalso + (WitnessC_PathEHJ == WitnessF_PathEHJ) andalso + (WitnessA_PathEHJ == WitnessC_PathEHJ) andalso + (WitnessI_PathEHJ > WitnessA_PathEHJ), + + %% for path k, h, j + %% i gets lower scaled reward + WitnessA_PathKHJ = maps:get(a, blockchain_test_reward_store:fetch(path_khj_witness_rewards)), + WitnessB_PathKHJ = maps:get(b, blockchain_test_reward_store:fetch(path_khj_witness_rewards)), + WitnessC_PathKHJ = maps:get(c, blockchain_test_reward_store:fetch(path_khj_witness_rewards)), + WitnessF_PathKHJ = maps:get(f, blockchain_test_reward_store:fetch(path_khj_witness_rewards)), + WitnessI_PathKHJ = maps:get(i, blockchain_test_reward_store:fetch(path_khj_witness_rewards)), + + true = (WitnessA_PathKHJ == WitnessB_PathKHJ) andalso + (WitnessC_PathKHJ == WitnessF_PathKHJ) andalso + (WitnessA_PathKHJ == WitnessC_PathKHJ) andalso + (WitnessI_PathKHJ < WitnessA_PathKHJ), + + %% for path g, h, j + %% i gets equal scaled reward + WitnessA_PathGHJ = maps:get(a, blockchain_test_reward_store:fetch(path_ghj_witness_rewards)), + WitnessB_PathGHJ = maps:get(b, blockchain_test_reward_store:fetch(path_ghj_witness_rewards)), + WitnessC_PathGHJ = maps:get(c, blockchain_test_reward_store:fetch(path_ghj_witness_rewards)), + WitnessF_PathGHJ = maps:get(f, blockchain_test_reward_store:fetch(path_ghj_witness_rewards)), + WitnessI_PathGHJ = maps:get(i, blockchain_test_reward_store:fetch(path_ghj_witness_rewards)), + + true = (WitnessA_PathGHJ == WitnessB_PathGHJ) andalso + (WitnessC_PathGHJ == WitnessF_PathGHJ) andalso + (WitnessA_PathGHJ == WitnessC_PathGHJ) andalso + (WitnessI_PathGHJ == WitnessA_PathGHJ), ok. %%-------------------------------------------------------------------- %% HELPER %%-------------------------------------------------------------------- - -run_test(Witnesses, Config) -> - ct:pal("Config: ~p", [Config]), - BaseDir = ?config(base_dir, Config), +run_challengees_test(Constructor, + Elem1, + Elem2, + Elem3, + Layer1Witnesses, + Layer2Witnesses, + Config) -> + GenesisMembers = ?config(genesis_members, Config), ConsensusMembers = ?config(consensus_members, Config), - BaseDir = ?config(base_dir, Config), Chain = ?config(chain, Config), TCName = ?config(tc_name, Config), - Store = ?config(store, Config), - ct:pal("store: ~p", [Store]), - - Ledger = blockchain:ledger(Chain), - Vars = blockchain_ledger_v1:snapshot_vars(Ledger), - ct:pal("Vars: ~p", [Vars]), + #{ gateway_names := GatewayNameMap, gateway_letter_to_addr := GatewayLetterToAddrMap } = cross_check_maps(Config), - AG = blockchain_ledger_v1:active_gateways(Ledger), + Challenger = maps:get(Constructor, GatewayLetterToAddrMap), + {_, {_, _, ChallengerSigFun}} = lists:keyfind(Challenger, 1, GenesisMembers), - GatewayAddrs = lists:sort(maps:keys(AG)), + GwElem1 = maps:get(Elem1, GatewayLetterToAddrMap), + GwElem2 = maps:get(Elem2, GatewayLetterToAddrMap), + GwElem3 = maps:get(Elem3, GatewayLetterToAddrMap), - AllGws = [a, b, c, d, e, f, g, h, i, j, k], + Rx1 = blockchain_poc_receipt_v1:new( + GwElem1, + 1000, + 10, + <<"first_rx">>, + p2p + ), + Rx2 = blockchain_poc_receipt_v1:new( + GwElem2, + 1000, + 10, + <<"second_rx">>, + radio + ), + Rx3 = blockchain_poc_receipt_v1:new( + GwElem3, + 1000, + 10, + <<"third_rx">>, + radio + ), - %% For crosscheck - GatewayNameMap = lists:foldl( - fun({Letter, A}, Acc) -> - maps:put(blockchain_utils:addr2name(A), Letter, Acc) + ConstructedWitnesses1 = lists:foldl( + fun(W, Acc) -> + WitnessGw = maps:get(W, GatewayLetterToAddrMap), + Witness = blockchain_poc_witness_v1:new( + WitnessGw, + 1001, + 10, + crypto:strong_rand_bytes(32), + 9, + 915, + 10, + "data_rate" + ), + [Witness | Acc] end, - #{}, - lists:zip(AllGws, GatewayAddrs) + [], + Layer1Witnesses ), - %% For crosscheck - GatewayLocMap = lists:foldl( - fun(A, Acc) -> - {ok, Gw} = blockchain_ledger_v1:find_gateway_info(A, Ledger), - GwLoc = blockchain_ledger_gateway_v2:location(Gw), - maps:put(blockchain_utils:addr2name(A), GwLoc, Acc) + ConstructedWitnesses2 = lists:foldl( + fun(W, Acc) -> + WitnessGw = maps:get(W, GatewayLetterToAddrMap), + Witness = blockchain_poc_witness_v1:new( + WitnessGw, + 1001, + 10, + crypto:strong_rand_bytes(32), + 9, + 915, + 10, + "data_rate" + ), + [Witness | Acc] end, - #{}, - GatewayAddrs + [], + Layer2Witnesses ), - %% For crosscheck - GatewayLetterToAddrMap = lists:foldl( - fun({Letter, A}, Acc) -> - maps:put(Letter, A, Acc) + %% Construct poc receipt txn + P1 = blockchain_poc_path_element_v1:new(GwElem1, Rx1, ConstructedWitnesses1), + P2 = blockchain_poc_path_element_v1:new(GwElem2, Rx2, ConstructedWitnesses2), + P3 = blockchain_poc_path_element_v1:new(GwElem3, Rx3, []), + + ct:pal("P1: ~p", [P1]), + ct:pal("P2: ~p", [P2]), + ct:pal("P3: ~p", [P3]), + + %% We'll consider all the witnesses to be "good quality" for the sake of testing + meck:expect(blockchain_txn_poc_request_v1, is_valid, fun(_, _) -> + ok + end), + meck:expect(blockchain_txn_poc_receipts_v1, is_valid, fun(_, _) -> ok end), + meck:expect(blockchain_txn_poc_receipts_v1, absorb, fun(_, _) -> ok end), + meck:expect(blockchain_txn_poc_receipts_v1, get_channels, fun(_, _) -> + {ok, lists:seq(1, 11)} + end), + meck:expect(blockchain_txn_poc_receipts_v1, good_quality_witnesses, + fun + (E, _) when E == P1 -> + ConstructedWitnesses1; + (E, _) when E == P2 -> + ConstructedWitnesses2; + (_, _) -> + [] + end), + meck:expect( + blockchain_txn_rewards_v1, + legit_witnesses, + fun + (_, _, _, E, _, _) when E == P1 -> + ConstructedWitnesses1; + (_, _, _, E, _, _) when E == P2 -> + ConstructedWitnesses2; + (_, _, _, _, _, _) -> + [] + end + ), + meck:expect(blockchain_txn_poc_receipts_v1, valid_witnesses, + fun + (E, _, _) when E == P1 -> + ConstructedWitnesses1; + (E, _, _) when E == P2 -> + ConstructedWitnesses2; + (_, _, _) -> + [] + end), + + + Secret = crypto:strong_rand_bytes(32), + OnionKeyHash = crypto:strong_rand_bytes(32), + BlockHash = crypto:strong_rand_bytes(32), + + RTxn0 = blockchain_txn_poc_request_v1:new(Challenger, + Secret, + OnionKeyHash, + BlockHash, + 10), + RTxn = blockchain_txn_poc_request_v1:sign(RTxn0, ChallengerSigFun), + ct:pal("RTxn: ~p", [RTxn]), + %% Construct a block for the poc request txn + {ok, Block2} = test_utils:create_block(ConsensusMembers, [RTxn], #{}, false), + ct:pal("Block2: ~p", [Block2]), + _ = blockchain_gossip_handler:add_block(Block2, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 2}, blockchain:height(Chain)), + + Txn0 = blockchain_txn_poc_receipts_v1:new( + Challenger, + Secret, + OnionKeyHash, + BlockHash, + [P1, P2, P3] + ), + Txn = blockchain_txn_poc_receipts_v1:sign(Txn0, ChallengerSigFun), + ct:pal("Txn: ~p", [Txn]), + + %% Construct a block for the poc receipt txn WITHOUT validation + {ok, Block3} = test_utils:create_block(ConsensusMembers, [Txn], #{}, false), + ct:pal("Block3: ~p", [Block3]), + _ = blockchain_gossip_handler:add_block(Block3, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 3}, blockchain:height(Chain)), + + %% Empty block + {ok, Block4} = test_utils:create_block(ConsensusMembers, []), + ct:pal("Block4: ~p", [Block4]), + _ = blockchain_gossip_handler:add_block(Block4, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 4}, blockchain:height(Chain)), + + %% Calculate rewards by hand + Start = 1, + End = 3, + {ok, Rewards} = blockchain_txn_rewards_v1:calculate_rewards(Start, End, Chain), + + ChallengeesRewards = lists:filter( + fun(R) -> + blockchain_txn_reward_v1:type(R) == poc_challengees end, - #{}, - lists:zip(AllGws, GatewayAddrs) + Rewards ), - %% For crosscheck - GatewayLetterLocMap = lists:foldl( - fun({Letter, A}, Acc) -> - {ok, Gw} = blockchain_ledger_v1:find_gateway_info(A, Ledger), - GwLoc = blockchain_ledger_gateway_v2:location(Gw), - maps:put(Letter, GwLoc, Acc) + WitnessRewards = lists:filter( + fun(R) -> + blockchain_txn_reward_v1:type(R) == poc_witnesses end, - #{}, - lists:zip(AllGws, GatewayAddrs) + Rewards ), + ChallengeesRewardsMap = + lists:foldl( + fun(R, Acc) -> + maps:put( + maps:get( + blockchain_utils:addr2name(blockchain_txn_reward_v1:gateway(R)), + GatewayNameMap + ), + blockchain_txn_reward_v1:amount(R), + Acc + ) + end, + #{}, + ChallengeesRewards + ), + + WitnessRewardsMap = + lists:foldl( + fun(R, Acc) -> + maps:put( + maps:get( + blockchain_utils:addr2name(blockchain_txn_reward_v1:gateway(R)), + GatewayNameMap + ), + blockchain_txn_reward_v1:amount(R), + Acc + ) + end, + #{}, + WitnessRewards + ), + + ct:pal("ChallengeesRewardsMap: ~p", [ChallengeesRewardsMap]), + ct:pal("WitnessRewardsMap: ~p", [WitnessRewardsMap]), + + ok = blockchain_test_reward_store:insert( + list_to_atom(atom_to_list(TCName) ++ "_witness_rewards"), + WitnessRewardsMap + ), + ok = blockchain_test_reward_store:insert( + list_to_atom(atom_to_list(TCName) ++ "_challengee_rewards"), + ChallengeesRewardsMap + ), + + ok. + + +run_vars_test(Witnesses, Config) -> + GenesisMembers = ?config(genesis_members, Config), + ConsensusMembers = ?config(consensus_members, Config), + Chain = ?config(chain, Config), + TCName = ?config(tc_name, Config), + #{ gateway_names := GatewayNameMap, gateway_letter_to_addr := GatewayLetterToAddrMap } = cross_check_maps(Config), + Challenger = maps:get(k, GatewayLetterToAddrMap), + {_, {_, _, ChallengerSigFun}} = lists:keyfind(Challenger, 1, GenesisMembers), GwA = maps:get(a, GatewayLetterToAddrMap), GwD = maps:get(d, GatewayLetterToAddrMap), - GwH = maps:get(h, GatewayLetterToAddrMap), + GwI = maps:get(i, GatewayLetterToAddrMap), Rx1 = blockchain_poc_receipt_v1:new( GwA, 1000, 10, - "first_rx", + <<"first_rx">>, p2p ), Rx2 = blockchain_poc_receipt_v1:new( GwD, 1000, 10, - "second_rx", + <<"second_rx">>, radio ), Rx3 = blockchain_poc_receipt_v1:new( - GwH, + GwI, 1000, 10, - "third_rx", + <<"third_rx">>, radio ), - ct:pal("Rx1: ~p", [Rx1]), - ct:pal("Rx2: ~p", [Rx2]), - ct:pal("Rx3: ~p", [Rx3]), - ConstructedWitnesses = lists:foldl( fun(W, Acc) -> WitnessGw = maps:get(W, GatewayLetterToAddrMap), @@ -319,7 +681,7 @@ run_test(Witnesses, Config) -> 9.8, 915.2, 10, - <<"data_rate">> + "data_rate" ), [Witness | Acc] end, @@ -327,8 +689,6 @@ run_test(Witnesses, Config) -> Witnesses ), - ct:pal("ConstructedWitnesses: ~p", [ConstructedWitnesses]), - %% We'll consider all the witnesses to be "good quality" for the sake of testing meck:expect( blockchain_txn_rewards_v1, @@ -337,6 +697,10 @@ run_test(Witnesses, Config) -> ConstructedWitnesses end ), + meck:expect(blockchain_txn_poc_request_v1, is_valid, fun(_, _) -> + ok + end), + meck:expect(blockchain_txn_poc_receipts_v1, is_valid, fun(_, _) -> ok end), meck:expect(blockchain_txn_poc_receipts_v1, absorb, fun(_, _) -> ok end), meck:expect(blockchain_txn_poc_receipts_v1, valid_witnesses, fun(_, _, _) -> ConstructedWitnesses @@ -347,39 +711,58 @@ run_test(Witnesses, Config) -> meck:expect(blockchain_txn_poc_receipts_v1, get_channels, fun(_, _) -> {ok, lists:seq(1, 11)} end), - meck:expect(blockchain_hex, destroy_memoization, fun() -> - true - end), + %% meck:expect(blockchain_hex, destroy_memoization, fun() -> + %% true + %% end), + + Secret = crypto:strong_rand_bytes(32), + OnionKeyHash = crypto:strong_rand_bytes(32), + BlockHash = crypto:strong_rand_bytes(32), + + RTxn0 = blockchain_txn_poc_request_v1:new(Challenger, + Secret, + OnionKeyHash, + BlockHash, + 10), + RTxn = blockchain_txn_poc_request_v1:sign(RTxn0, ChallengerSigFun), + ct:pal("RTxn: ~p", [RTxn]), + %% Construct a block for the poc request txn + {ok, Block2} = test_utils:create_block(ConsensusMembers, [RTxn], #{}, false), + ct:pal("Block2: ~p", [Block2]), + _ = blockchain_gossip_handler:add_block(Block2, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 2}, blockchain:height(Chain)), + %% Construct poc receipt txn P1 = blockchain_poc_path_element_v1:new(GwA, Rx1, []), P2 = blockchain_poc_path_element_v1:new(GwD, Rx2, ConstructedWitnesses), - P3 = blockchain_poc_path_element_v1:new(GwH, Rx3, []), + P3 = blockchain_poc_path_element_v1:new(GwI, Rx3, []), ct:pal("P1: ~p", [P1]), ct:pal("P2: ~p", [P2]), ct:pal("P3: ~p", [P3]), - Txn = blockchain_txn_poc_receipts_v1:new( + Txn0 = blockchain_txn_poc_receipts_v1:new( Challenger, - <<"secret">>, - <<"onion_key_hash">>, - <<"block_hash">>, + Secret, + OnionKeyHash, + BlockHash, [P1, P2, P3] ), + Txn = blockchain_txn_poc_receipts_v1:sign(Txn0, ChallengerSigFun), ct:pal("Txn: ~p", [Txn]), %% Construct a block for the poc receipt txn WITHOUT validation - {ok, Block2} = test_utils:create_block(ConsensusMembers, [Txn], #{}, false), - ct:pal("Block2: ~p", [Block2]), - _ = blockchain_gossip_handler:add_block(Block2, Chain, self(), blockchain_swarm:swarm()), - ?assertEqual({ok, 2}, blockchain:height(Chain)), - - %% Empty block - {ok, Block3} = test_utils:create_block(ConsensusMembers, []), + {ok, Block3} = test_utils:create_block(ConsensusMembers, [Txn], #{}, false), ct:pal("Block3: ~p", [Block3]), _ = blockchain_gossip_handler:add_block(Block3, Chain, self(), blockchain_swarm:swarm()), ?assertEqual({ok, 3}, blockchain:height(Chain)), + %% Empty block + {ok, Block4} = test_utils:create_block(ConsensusMembers, []), + ct:pal("Block4: ~p", [Block4]), + _ = blockchain_gossip_handler:add_block(Block4, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 4}, blockchain:height(Chain)), + %% Calculate rewards by hand Start = 1, End = 3, @@ -431,10 +814,6 @@ run_test(Witnesses, Config) -> WitnessRewards ), - %% Theoretically, gateways J, K should have higher witness rewards than B, C, E, F, G, I - ct:pal("GatewayNameMap: ~p", [GatewayNameMap]), - ct:pal("GatewayLocMap: ~p", [GatewayLocMap]), - ct:pal("GatewayLetterLocMap: ~p", [GatewayLetterLocMap]), ct:pal("ChallengeesRewardsMap: ~p", [ChallengeesRewardsMap]), ct:pal("WitnessRewardsMap: ~p", [WitnessRewardsMap]), @@ -449,6 +828,59 @@ run_test(Witnesses, Config) -> ok. +cross_check_maps(Config) -> + Chain = ?config(chain, Config), + Ledger = blockchain:ledger(Chain), + AG = blockchain_ledger_v1:active_gateways(Ledger), + GatewayAddrs = lists:sort(maps:keys(AG)), + AllGws = [a, b, c, d, e, f, g, h, i, j, k], + GatewayNameMap = lists:foldl( + fun({Letter, A}, Acc) -> + maps:put(blockchain_utils:addr2name(A), Letter, Acc) + end, + #{}, + lists:zip(AllGws, GatewayAddrs) + ), + + %% For crosscheck + GatewayLocMap = lists:foldl( + fun(A, Acc) -> + {ok, Gw} = blockchain_ledger_v1:find_gateway_info(A, Ledger), + GwLoc = blockchain_ledger_gateway_v2:location(Gw), + maps:put(blockchain_utils:addr2name(A), GwLoc, Acc) + end, + #{}, + GatewayAddrs + ), + + %% For crosscheck + GatewayLetterToAddrMap = lists:foldl( + fun({Letter, A}, Acc) -> + maps:put(Letter, A, Acc) + end, + #{}, + lists:zip(AllGws, GatewayAddrs) + ), + + %% For crosscheck + GatewayLetterLocMap = lists:foldl( + fun({Letter, A}, Acc) -> + {ok, Gw} = blockchain_ledger_v1:find_gateway_info(A, Ledger), + GwLoc = blockchain_ledger_gateway_v2:location(Gw), + maps:put(Letter, GwLoc, Acc) + end, + #{}, + lists:zip(AllGws, GatewayAddrs) + ), + + #{ + gateway_names => GatewayNameMap, + gateway_locs => GatewayLocMap, + gateway_letter_to_addr => GatewayLetterToAddrMap, + gateway_letter_to_loc => GatewayLetterLocMap + }. + + hip15_vars() -> #{ %% configured on chain @@ -478,70 +910,250 @@ hip17_vars() -> }. known_locations() -> + [631210990515645439, + 631210990515609087, + 631210990516667903, + 631210990528935935, + 631210990528385535, + 631210990528546815, + 631210990529462783, + 631210990529337343, + 631210990524024831, + 631210990524753919, + 631210990525267455 + ]. - %% NameToPubkeyBin = - %% #{ - %% "rare-amethyst-reindeer" => libp2p_crypto:b58_to_bin("112VfPXs1WTRjp24WkbbjQmbFNhb5ptot7gpGwJULm4SRiacjuvW"), - %% "cold-canvas-duck" => libp2p_crypto:b58_to_bin("112ciDxDUBwJZs5YjjPWJWKGwGtUtdJdxSgDAYJPhu9fHw4sgeQy"), - %% "melted-tangelo-aphid" => libp2p_crypto:b58_to_bin("112eNuzPYSeeo3tqNDidr2gPysz7QtLkePkY5Yn1V7ddNPUDN6p5"), - %% "early-lime-rat" => libp2p_crypto:b58_to_bin("112Xr4ZtiNbeh8wfiWTYfeo7KwBbXwvx5F2LPdNTC8wp8q4EQCAm"), - %% "flat-lilac-shrimp" => libp2p_crypto:b58_to_bin("112euXBKmLzUAfyi7FaYRxRpcH5RmfPKprV3qEyHCTt8nqwyVFYo"), - %% "harsh-sandstone-stork" => libp2p_crypto:b58_to_bin("11UFysjjP9W8S7ZV54iK7L6HpkxkHrSPRm4rKkWq22cStYYhDhM"), - %% "pet-pewter-lobster" => libp2p_crypto:b58_to_bin("112LYrRkJX32jVNsuAzt9kDqrddXqWrwpG8N5QX2hELvzf8JJZbw"), - %% "amateur-tan-monkey" => libp2p_crypto:b58_to_bin("112bQKSN3TiaYMrsjNKGZotd14QPi7DB37FeV88rmVMgP4MgTK9q"), - %% "clean-wooden-zebra" => libp2p_crypto:b58_to_bin("112AT5baYcYG6yHchYa9xnkqNJ4cXbgxCh8i8nvnfZeewAHJ8zKc"), - %% "odd-champagne-nuthatch" => libp2p_crypto:b58_to_bin("112fDV4b5FqSgcnu3F592RauuNo5HkuPzfETM7WJ9AfCdaCo9sLk"), - %% "abundant-grape-butterfly" => libp2p_crypto:b58_to_bin("112pdh3waHFbu3XqtCWwbw9xEtYtUEvbqzgSVbEoENBRQznj9Tuy") - %% }, - - %% Locs = - %% #{ - %% "rare-amethyst-reindeer" => 631786582666296319, - %% "cold-canvas-duck" => 631786582410363903, - %% "melted-tangelo-aphid" => 631786582655116287, - %% "early-lime-rat" => 631786582659491327, - %% "flat-lilac-shrimp" => 631786581941828607, - %% "harsh-sandstone-stork" => 631786581946850303, - %% "pet-pewter-lobster" => 631786581906280959, - %% "amateur-tan-monkey" => 631786581937244159, - %% "clean-wooden-zebra" => 631786581846989823, - %% "odd-champagne-nuthatch" => 631786581944091647, - %% "abundant-grape-butterfly" => 631786582694056959 - %% }, - - %% [631786581906280959, - %% 631786582666296319, - %% 631786582410363903, - %% 631786582694056959, - %% 631786582655116287, - %% 631786582659491327, - %% 631786581941828607, - %% 631786581937244159, - %% 631786581946850303, - %% 631786581846989823, - %% 631786581944091647], - - - %% [631211351866199551, - %% 631211351866199551, - %% 631211351866084351, - %% 631211351866223615, - %% 631211351866300415, - %% 631211351866239999, - %% 631211351866165759, - %% 631211351866165247, - %% 631211351866289663, - %% 631211351865407487, - %% 631211351865991679], - - [631211351866199551, - 631211351866199551, - 631211351866199551, - 631211351866199551, - 631211351866199551, - 631211351866199551, - 631211351866199551, - 631211351866199551, - 631211351866199551, - 631211351866199551, - 631211351866199551]. +static_keys() -> + %% These static keys are only generated to ensure we get consistent results + %% between successive test runs + [{<<0,135,35,145,156,56,241,143,56,203,33,59,137,227,51,220,158,75,110, + 92,79,47,197,189,61,2,166,40,158,85,94,187,210>>, + {ecc_compact,{{'ECPoint',<<4,135,35,145,156,56,241,143,56,203,33,59, + 137,227,51,220,158,75,110,92,79,47,197,189, + 61,2,166,40,158,85,94,187,210,23,124,197, + 126,82,136,156,224,77,29,244,7,181,54,27, + 193,238,247,20,220,223,82,172,29,184,166, + 244,80,180,158,234,200>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<156,174,164,201,194,229,247,253,190,55, + 192,6,178,54,77,8,107,207,119,165,225, + 56,57,77,56,93,18,7,204,3,87,19>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,135,35,145,156,56,241,143,56,203, + 33,59,137,227,51,220,158,75,110,92, + 79,47,197,189,61,2,166,40,158,85,94, + 187,210,23,124,197,126,82,136,156, + 224,77,29,244,7,181,54,27,193,238, + 247,20,220,223,82,172,29,184,166,244, + 80,180,158,234,200>>}}}, + {<<0,20,71,124,234,252,184,6,227,161,188,190,47,137,191,118,55,90,107, + 76,18,110,125,250,200,219,154,35,120,32,13,85,162>>, + {ecc_compact,{{'ECPoint',<<4,20,71,124,234,252,184,6,227,161,188,190, + 47,137,191,118,55,90,107,76,18,110,125,250, + 200,219,154,35,120,32,13,85,162,82,177,64, + 62,209,191,108,219,71,95,159,45,165,110,57, + 225,131,208,229,15,227,239,68,150,156,254, + 184,111,119,196,72,157>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<74,195,136,8,5,64,160,74,239,70,204,88, + 28,125,218,23,158,45,29,211,216,145,16, + 44,78,66,232,65,60,96,37,195>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,20,71,124,234,252,184,6,227,161, + 188,190,47,137,191,118,55,90,107,76, + 18,110,125,250,200,219,154,35,120,32, + 13,85,162,82,177,64,62,209,191,108, + 219,71,95,159,45,165,110,57,225,131, + 208,229,15,227,239,68,150,156,254, + 184,111,119,196,72,157>>}}}, + {<<0,217,189,89,225,39,191,180,16,213,28,126,134,2,140,86,174,237,57, + 197,104,123,176,138,216,163,253,140,2,4,159,237,17>>, + {ecc_compact,{{'ECPoint',<<4,217,189,89,225,39,191,180,16,213,28,126, + 134,2,140,86,174,237,57,197,104,123,176, + 138,216,163,253,140,2,4,159,237,17,100,96, + 191,118,251,152,42,105,120,182,220,31,13, + 120,76,56,254,170,50,153,63,47,84,160,68, + 36,156,45,187,209,160,81>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<225,210,133,73,145,176,53,145,226,86,23, + 195,148,179,16,42,71,247,197,165,158, + 144,151,227,103,187,209,110,2,16,100,99>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,217,189,89,225,39,191,180,16,213, + 28,126,134,2,140,86,174,237,57,197, + 104,123,176,138,216,163,253,140,2,4, + 159,237,17,100,96,191,118,251,152,42, + 105,120,182,220,31,13,120,76,56,254, + 170,50,153,63,47,84,160,68,36,156,45, + 187,209,160,81>>}}}, + {<<0,215,157,27,147,66,217,10,140,181,194,91,108,130,23,111,221,203,186, + 194,157,244,168,53,45,184,162,228,141,214,155,106,122>>, + {ecc_compact,{{'ECPoint',<<4,215,157,27,147,66,217,10,140,181,194,91, + 108,130,23,111,221,203,186,194,157,244,168, + 53,45,184,162,228,141,214,155,106,122,38, + 162,55,138,90,19,132,142,109,40,39,237,77, + 117,34,14,160,114,41,62,104,54,56,240,115, + 124,9,53,189,251,42,56>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<77,233,24,124,91,70,54,186,28,49,82,177, + 176,200,8,68,211,204,31,128,74,142,24, + 118,112,207,51,86,10,74,155,139>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,215,157,27,147,66,217,10,140,181, + 194,91,108,130,23,111,221,203,186, + 194,157,244,168,53,45,184,162,228, + 141,214,155,106,122,38,162,55,138,90, + 19,132,142,109,40,39,237,77,117,34, + 14,160,114,41,62,104,54,56,240,115, + 124,9,53,189,251,42,56>>}}}, + {<<0,85,63,44,227,26,250,122,155,247,250,201,91,215,217,210,181,152,209, + 90,103,54,116,57,145,2,191,107,135,227,150,188,139>>, + {ecc_compact,{{'ECPoint',<<4,85,63,44,227,26,250,122,155,247,250,201, + 91,215,217,210,181,152,209,90,103,54,116, + 57,145,2,191,107,135,227,150,188,139,106, + 164,125,200,121,31,25,3,125,231,89,189,212, + 151,43,237,167,194,1,145,180,132,128,149, + 213,142,55,113,43,57,129,192>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<241,178,164,75,59,239,80,187,86,100,0, + 137,105,108,64,161,36,76,103,226,66,241, + 42,114,119,131,125,203,205,2,213,21>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,85,63,44,227,26,250,122,155,247, + 250,201,91,215,217,210,181,152,209, + 90,103,54,116,57,145,2,191,107,135, + 227,150,188,139,106,164,125,200,121, + 31,25,3,125,231,89,189,212,151,43, + 237,167,194,1,145,180,132,128,149, + 213,142,55,113,43,57,129,192>>}}}, + {<<0,33,183,166,74,151,68,17,14,58,106,91,30,171,149,116,42,54,136,187, + 6,135,149,78,44,132,144,224,168,180,185,5,210>>, + {ecc_compact,{{'ECPoint',<<4,33,183,166,74,151,68,17,14,58,106,91,30, + 171,149,116,42,54,136,187,6,135,149,78,44, + 132,144,224,168,180,185,5,210,8,205,10,58, + 37,17,206,158,32,200,182,231,53,43,66,110, + 7,107,125,127,244,91,98,235,213,107,130, + 177,229,189,26,225>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<35,37,85,183,224,18,148,11,77,133,138, + 152,248,222,56,7,8,70,251,212,124,223, + 107,122,184,18,15,60,254,173,172,18>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,33,183,166,74,151,68,17,14,58,106, + 91,30,171,149,116,42,54,136,187,6, + 135,149,78,44,132,144,224,168,180, + 185,5,210,8,205,10,58,37,17,206,158, + 32,200,182,231,53,43,66,110,7,107, + 125,127,244,91,98,235,213,107,130, + 177,229,189,26,225>>}}}, + {<<0,197,206,105,167,43,204,77,56,215,206,79,130,83,194,243,95,100,232, + 161,135,166,145,34,142,103,155,65,147,209,189,13,145>>, + {ecc_compact,{{'ECPoint',<<4,197,206,105,167,43,204,77,56,215,206,79, + 130,83,194,243,95,100,232,161,135,166,145, + 34,142,103,155,65,147,209,189,13,145,120,0, + 190,129,210,122,118,155,125,166,201,50,78, + 61,217,80,236,99,106,75,181,30,27,230,173, + 173,102,84,25,102,28,126>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<248,151,148,106,71,235,30,171,175,38,61, + 208,228,196,194,195,249,95,180,188,95, + 132,216,225,68,184,114,177,226,242,21,60>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,197,206,105,167,43,204,77,56,215, + 206,79,130,83,194,243,95,100,232,161, + 135,166,145,34,142,103,155,65,147, + 209,189,13,145,120,0,190,129,210,122, + 118,155,125,166,201,50,78,61,217,80, + 236,99,106,75,181,30,27,230,173,173, + 102,84,25,102,28,126>>}}}, + {<<0,7,94,141,107,189,125,163,87,153,196,200,77,40,78,50,238,22,1,154, + 70,45,135,148,16,46,120,188,198,164,147,250,255>>, + {ecc_compact,{{'ECPoint',<<4,7,94,141,107,189,125,163,87,153,196,200, + 77,40,78,50,238,22,1,154,70,45,135,148,16, + 46,120,188,198,164,147,250,255,19,149,195, + 194,244,103,75,60,21,25,210,102,160,221, + 218,228,176,202,38,39,78,110,184,59,52,196, + 95,173,105,168,140,210>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<208,185,203,219,249,155,80,235,230,243, + 229,64,55,110,230,34,135,106,11,22,26, + 202,149,11,135,154,242,158,9,77,136,242>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,7,94,141,107,189,125,163,87,153, + 196,200,77,40,78,50,238,22,1,154,70, + 45,135,148,16,46,120,188,198,164,147, + 250,255,19,149,195,194,244,103,75,60, + 21,25,210,102,160,221,218,228,176, + 202,38,39,78,110,184,59,52,196,95, + 173,105,168,140,210>>}}}, + {<<0,161,224,241,247,215,177,248,246,170,70,111,93,20,168,111,142,225, + 183,129,50,237,242,215,38,34,0,224,216,228,118,55,26>>, + {ecc_compact,{{'ECPoint',<<4,161,224,241,247,215,177,248,246,170,70, + 111,93,20,168,111,142,225,183,129,50,237, + 242,215,38,34,0,224,216,228,118,55,26,68, + 26,209,120,63,198,91,107,11,223,80,59,88, + 34,239,206,159,182,46,177,249,154,53,8,38, + 195,129,102,176,32,85,201>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<180,123,47,194,133,184,161,103,6,218, + 189,247,36,157,70,102,114,5,199,223,38, + 24,244,74,248,111,229,69,30,232,234,205>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,161,224,241,247,215,177,248,246, + 170,70,111,93,20,168,111,142,225,183, + 129,50,237,242,215,38,34,0,224,216, + 228,118,55,26,68,26,209,120,63,198, + 91,107,11,223,80,59,88,34,239,206, + 159,182,46,177,249,154,53,8,38,195, + 129,102,176,32,85,201>>}}}, + {<<0,181,132,129,89,193,104,228,73,203,137,46,161,153,156,56,205,253, + 243,206,109,218,93,13,9,77,222,143,147,148,135,39,15>>, + {ecc_compact,{{'ECPoint',<<4,181,132,129,89,193,104,228,73,203,137,46, + 161,153,156,56,205,253,243,206,109,218,93, + 13,9,77,222,143,147,148,135,39,15,122,11, + 108,114,34,18,8,83,94,141,159,31,164,88, + 209,127,192,66,45,132,137,86,40,57,122,188, + 185,86,99,180,161,224>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<155,29,3,116,69,125,244,121,73,22,179, + 49,210,8,187,245,179,70,6,58,3,60,12, + 136,25,186,144,133,58,236,232,160>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,181,132,129,89,193,104,228,73,203, + 137,46,161,153,156,56,205,253,243, + 206,109,218,93,13,9,77,222,143,147, + 148,135,39,15,122,11,108,114,34,18,8, + 83,94,141,159,31,164,88,209,127,192, + 66,45,132,137,86,40,57,122,188,185, + 86,99,180,161,224>>}}}, + {<<0,156,114,72,115,65,211,12,113,160,134,127,252,250,62,10,149,32,182, + 30,19,158,41,162,182,224,15,48,57,27,13,50,200>>, + {ecc_compact,{{'ECPoint',<<4,156,114,72,115,65,211,12,113,160,134,127, + 252,250,62,10,149,32,182,30,19,158,41,162, + 182,224,15,48,57,27,13,50,200,108,73,193, + 233,44,47,102,122,37,188,76,253,248,32,143, + 166,59,54,47,46,239,151,157,211,72,75,185, + 81,203,125,20,50>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<32,249,3,92,124,248,0,146,225,224,17,2, + 87,54,7,126,165,185,28,110,196,141,58, + 35,250,244,162,224,158,40,0,28>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,156,114,72,115,65,211,12,113,160, + 134,127,252,250,62,10,149,32,182,30, + 19,158,41,162,182,224,15,48,57,27, + 13,50,200,108,73,193,233,44,47,102, + 122,37,188,76,253,248,32,143,166,59, + 54,47,46,239,151,157,211,72,75,185, + 81,203,125,20,50>>}}}]. diff --git a/test/test_utils.erl b/test/test_utils.erl index 3a68476a68..02f2c58bea 100644 --- a/test/test_utils.erl +++ b/test/test_utils.erl @@ -7,7 +7,8 @@ -export([ init/1, init/2, init_chain/2, init_chain/3, init_chain/4, - init_chain_with_fixed_locations/5, + init_chain_with_fixed_locations/4, + generate_plain_keys/2, generate_keys/1, generate_keys/2, wait_until/1, wait_until/3, create_block/2, create_block/3, create_block/4, @@ -107,10 +108,9 @@ init_chain(Balance, GenesisMembers, ExtraVars) when is_list(GenesisMembers), is_ ?assertEqual({ok, 1}, blockchain:height(Chain)), {ok, GenesisMembers, GenesisBlock, ConsensusMembers, Keys}. -init_chain_with_fixed_locations(Balance, {PrivKey, PubKey}, InConsensus, Locations, ExtraVars) when is_list(Locations), is_map(ExtraVars) -> - - GenesisMembers = init_genesis_members({PrivKey, PubKey}, InConsensus), - +init_chain_with_fixed_locations(Balance, GenesisMembers, Locations, ExtraVars) when is_list(Locations), + is_list(GenesisMembers), + is_map(ExtraVars) -> % Create genesis block {InitialVars, Keys} = blockchain_ct_utils:create_vars(ExtraVars), @@ -147,7 +147,17 @@ init_chain_with_fixed_locations(Balance, {PrivKey, PubKey}, InConsensus, Locatio ?assertEqual({ok, blockchain_block:hash_block(GenesisBlock)}, blockchain:genesis_hash(Chain)), ?assertEqual({ok, GenesisBlock}, blockchain:genesis_block(Chain)), ?assertEqual({ok, 1}, blockchain:height(Chain)), - {ok, GenesisMembers, GenesisBlock, ConsensusMembers, Keys}. + {ok, GenesisBlock, ConsensusMembers, Keys}. + +generate_plain_keys(N, Type) -> + lists:foldl( + fun(_, Acc) -> + #{public := PubKey, secret := PrivKey} = libp2p_crypto:generate_keys(Type), + [{libp2p_crypto:pubkey_to_bin(PubKey), PubKey, PrivKey}|Acc] + end + ,[] + ,lists:seq(1, N) + ). generate_keys(N) -> generate_keys(N, ecc_compact). From db4ef8ca70f4ea91595c0f5d2fc160d4bbbd0438 Mon Sep 17 00:00:00 2001 From: Evan Vigil-McClanahan Date: Wed, 9 Dec 2020 14:32:56 -0800 Subject: [PATCH 15/18] fix dialyzer --- src/ledger/v1/blockchain_ledger_snapshot_v1.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ledger/v1/blockchain_ledger_snapshot_v1.erl b/src/ledger/v1/blockchain_ledger_snapshot_v1.erl index f7a8fc111f..bd175aaf9b 100644 --- a/src/ledger/v1/blockchain_ledger_snapshot_v1.erl +++ b/src/ledger/v1/blockchain_ledger_snapshot_v1.erl @@ -1077,7 +1077,6 @@ v4_to_v3(#blockchain_snapshot_v4{ v5_to_v4(#{ version := v5, current_height := CurrHeight, - transaction_fee := 0, consensus_members := ConsensusMembers, election_height := ElectionHeight, @@ -1118,6 +1117,8 @@ v5_to_v4(#{ current_height = CurrHeight, consensus_members = ConsensusMembers, + transaction_fee = 0, + election_height = ElectionHeight, election_epoch = ElectionEpoch, From 3f5d9c8a874976aa1cbdd2f278be8273de9fbb65 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Wed, 9 Dec 2020 14:40:01 -0800 Subject: [PATCH 16/18] Review comment --- src/blockchain_hex.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl index afd141410a..2fed6709af 100644 --- a/src/blockchain_hex.erl +++ b/src/blockchain_hex.erl @@ -221,7 +221,7 @@ densities(H3Root, VarMap, Locations, Ledger) -> [Head | Tail] = lists:seq(UpperBoundRes, LowerBoundRes, -1), - %% find parent hexs to all hotspots at highest resolution in chain variables + %% find parent hexes to all hotspots at highest resolution in chain variables {ParentHexes, InitialDensities} = maps:fold( fun(Hex, GWs, {HAcc, MAcc}) -> From 6e53fd9eae9d4a59d4db719b7732853b2c88ff49 Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Wed, 9 Dec 2020 15:38:49 -0800 Subject: [PATCH 17/18] Dont do unnecessary calc on old reward version --- .../v1/blockchain_txn_rewards_v1.erl | 27 +++---------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index d9b6440d88..00fa601767 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -646,18 +646,8 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; _ -> - case poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) of - {error, _} -> - %% Old behavior - maps:put(Challengee, I+1, Acc0); - {ok, ToAdd} -> - TxScale = maybe_calc_tx_scale(Challengee, - DensityTgtRes, - ChallengeeLoc, - VarMap, - Ledger), - maps:put(Challengee, I+(ToAdd * TxScale), Acc0) - end + %% Old behavior + maps:put(Challengee, I+1, Acc0) end, poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1); p2p -> @@ -685,18 +675,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; true -> - case poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) of - {error, _} -> - %% Old behavior - maps:put(Challengee, I+1, Acc0); - {ok, ToAdd} -> - TxScale = maybe_calc_tx_scale(Challengee, - DensityTgtRes, - ChallengeeLoc, - VarMap, - Ledger), - maps:put(Challengee, I+(ToAdd * TxScale), Acc0) - end + maps:put(Challengee, I+1, Acc0) end, poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1) end From 9032342e3f514cda698ef096879901f50f845b0f Mon Sep 17 00:00:00 2001 From: Rahul Garg Date: Wed, 9 Dec 2020 16:01:32 -0800 Subject: [PATCH 18/18] Review comment, cleanup --- .../v1/blockchain_txn_rewards_v1.erl | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index 00fa601767..3c1fa50003 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -777,24 +777,18 @@ poc_witnesses_rewards(Transactions, lists:foldl( fun(WitnessRecord, Acc2) -> Challengee = blockchain_poc_path_element_v1:challengee(Elem), - case blockchain_ledger_v1:find_gateway_info(Challengee, Ledger) of - {ok, ChallengeeGw} -> - case blockchain_ledger_gateway_v2:location(ChallengeeGw) of - undefined -> - Acc2; - ChallengeeLoc -> - Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), - %% The witnesses get scaled by the value of their transmitters - RxScale = blockchain_hex:scale(ChallengeeLoc, - VarMap, - D, - Ledger), - I = maps:get(Witness, Acc2, 0), - maps:put(Witness, I+(ToAdd*RxScale), Acc2) - end; - _ -> - Acc2 - end + %% This must always be {ok, ...} + {ok, ChallengeeGw} = blockchain_ledger_v1:find_gateway_info(Challengee, Ledger), + %% Challengee must have a location + ChallengeeLoc = blockchain_ledger_gateway_v2:location(ChallengeeGw), + Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), + %% The witnesses get scaled by the value of their transmitters + RxScale = blockchain_hex:scale(ChallengeeLoc, + VarMap, + D, + Ledger), + I = maps:get(Witness, Acc2, 0), + maps:put(Witness, I+(ToAdd*RxScale), Acc2) end, Acc1, ValidWitnesses