diff --git a/eqc/eqc_utils.erl b/eqc/eqc_utils.erl index b72403ea8f..2a53f1e0b5 100644 --- a/eqc/eqc_utils.erl +++ b/eqc/eqc_utils.erl @@ -5,7 +5,8 @@ ledger/1, name/1, maybe_output_paths/3, - ledger_vars/1 + ledger_vars/1, + big_witness_hotspots/0 ]). find_challenger(ChallengerIndex, ActiveGateways) -> @@ -47,8 +48,8 @@ name(PubkeyBin) -> Name. ledger(ExtraVars) -> - %% Ledger at height: 194196 - %% ActiveGateway Count: 3023 + %% Ledger at height: 481929 + %% ActiveGateway Count: 8000 {ok, Dir} = file:get_cwd(), %% Ensure priv dir exists PrivDir = filename:join([Dir, "priv"]), @@ -72,6 +73,15 @@ ledger(ExtraVars) -> ledger_vars(Ledger) -> blockchain_utils:vars_binary_keys_to_atoms(maps:from_list(blockchain_ledger_v1:snapshot_vars(Ledger))). +big_witness_hotspots() -> + {ok, Dir} = file:get_cwd(), + %% Ensure priv dir exists + PrivDir = filename:join([Dir, "priv"]), + ok = filelib:ensure_dir(PrivDir ++ "/"), + BigWitnessHotspotFile = filename:join([PrivDir, "big_witness_hotspots"]), + {ok, LBin} = file:read_file(BigWitnessHotspotFile), + binary_to_term(LBin). + extract_ledger_tar(PrivDir, LedgerTar) -> case filelib:is_file(LedgerTar) of true -> @@ -87,7 +97,7 @@ extract_ledger_tar(PrivDir, LedgerTar) -> false -> %% ledger tar file not found, download & extract ok = ssl:start(), - {ok, {{_, 200, "OK"}, _, Body}} = httpc:request("https://blockchain-core.s3-us-west-1.amazonaws.com/ledger.tar.gz"), + {ok, {{_, 200, "OK"}, _, Body}} = httpc:request("https://blockchain-core.s3-us-west-1.amazonaws.com/ledger-481929.tar.gz"), ok = file:write_file(filename:join([PrivDir, "ledger.tar.gz"]), Body), erl_tar:extract(LedgerTar, [compressed, {cwd, PrivDir}]) end. diff --git a/eqc/path_v4_eqc.erl b/eqc/path_v4_eqc.erl index 6e0548b5bd..7b3f393697 100644 --- a/eqc/path_v4_eqc.erl +++ b/eqc/path_v4_eqc.erl @@ -9,11 +9,12 @@ [ledger/1, dead_hotspots/0, find_challenger/2, - ledger_vars/1 - %% maybe_output_paths/3 + ledger_vars/1, + big_witness_hotspots/0, + maybe_output_paths/3 ]). --export([prop_path_check/0]). +-export([prop_path_check/0, prop_path_top_200_witness_check/0]). prop_path_check() -> ?FORALL({Hash, PathLimit, ChallengerIndex}, @@ -77,7 +78,7 @@ prop_path_check() -> end, PathLength = length(Path), - %% ok = maybe_output_paths(TargetPubkeyBin, Path, Time), + ok = maybe_output_paths(TargetPubkeyBin, Path, Time), %% Checks: %% - honor path limit @@ -113,6 +114,107 @@ prop_path_check() -> ) end). +prop_path_top_200_witness_check() -> + ?FORALL({Hash, PathLimit, ChallengerIndex, TargetPubkeyBin}, + {gen_hash(), gen_path_limit(), gen_challenger_index(), gen_big_witness_target()}, + begin + {ok, GWCache} = blockchain_gateway_cache:start_link(), + Ledger = ledger(poc_v8_vars()), + application:set_env(blockchain, disable_score_cache, true), + {ok, _Pid} = blockchain_score_cache:start_link(), + ActiveGateways = blockchain_ledger_v1:active_gateways(Ledger), + LedgerVars = ledger_vars(Ledger), + + %% use this to artificially constrain path limit for testing + %% PathLimit = 5, + + %% use this to make timings more realistic for pi + %% disks, set to 1 or 2 + application:set_env(blockchain, find_gateway_sim_delay, 0), + + %% Overwrite poc_path_limit for checking generated path limits + Vars = maps:put(poc_path_limit, PathLimit, LedgerVars), + + %% Find some challenger + {ChallengerPubkeyBin, _ChallengerLoc} = find_challenger(ChallengerIndex, ActiveGateways), + + {ok, {_, TargetRandState}} = blockchain_poc_target_v3:target(ChallengerPubkeyBin, Hash, Ledger, Vars), + {Time, Path} = timer:tc(fun() -> + blockchain_poc_path_v4:build(TargetPubkeyBin, + TargetRandState, + Ledger, + block_time(), + Vars) + end), + + %% change the default to true to collect and display + %% time stats at each run + case application:get_env(blockchain, collect_stats, false) of + true -> + Runs = + case get(runs) of + undefined -> + put(runs, [Time]), + [Time]; + R -> + R1 = [Time | R], + put(runs, R1), + R1 + end, + + L = length(Runs), + Avg = trunc(lists:sum(Runs) / L), + Med = case L < 5 of + true -> no_med; + _ -> lists:nth((L div 2), lists:sort(Runs)) + end, + Max = lists:max(Runs), + + io:fwrite(standard_error, "build took avg ~p med ~p max ~p time ~p us\n", [Avg, Med, Max, Time]); + false -> + ok + end, + PathLength = length(Path), + + ok = maybe_output_paths(TargetPubkeyBin, Path, Time), + + %% Checks: + %% - honor path limit + %% - atleast one element in path + %% - target is always in path + %% - we never go back to the same h3 index in path + %% - check next hop is a witness of previous gateway + C1 = PathLength =< PathLimit andalso PathLength >= 1, + C2 = length(Path) == length(lists:usort(Path)), + C3 = lists:member(TargetPubkeyBin, Path), + C4 = check_path_h3_indices(Path, ActiveGateways), + C5 = check_next_hop(Path, ActiveGateways), + C6 = check_target_and_path_members_not_dead(TargetPubkeyBin, Path), + + blockchain_ledger_v1:close(Ledger), + blockchain_score_cache:stop(), + gen_server:stop(GWCache), + + ?WHENFAIL(begin + blockchain_ledger_v1:close(Ledger), + blockchain_score_cache:stop(), + gen_server:stop(GWCache) + end, + %% TODO: split into multiple verifiers instead of a single consolidated one + conjunction([ + {verify_path_limit, C1}, + {verify_minimum_one_element, C2}, + {verify_target_in_path, C3}, + {verify_non_vw_path, C4}, + {verify_next_hop_is_witness, C5}, + {verify_not_dead, C6} + ]) + ) + end). + +gen_big_witness_target() -> + elements(big_witness_hotspots()). + gen_path_limit() -> elements([3, 4, 5, 6, 7]). diff --git a/priv/big_witness_hotspots b/priv/big_witness_hotspots new file mode 100644 index 0000000000..fba761630e Binary files /dev/null and b/priv/big_witness_hotspots differ