Extend memory checks and profiling in repl test

OpenRiak · Jul 8, 2024 · 61e4d89 · 61e4d89
1 parent 39a000d
commit 61e4d89
Show file tree

Hide file tree

Showing 2 changed files with 63 additions and 14 deletions.
diff --git a/tests/general_api_perf.erl b/tests/general_api_perf.erl
@@ -20,7 +20,7 @@
 %% of that API activity
 
 -module(general_api_perf).
--export([confirm/0]).
+-export([confirm/0, spawn_profile_fun/1]).
 
 -import(secondary_index_tests, [http_query/3, pb_query/3]).
 -include_lib("eunit/include/eunit.hrl").
@@ -38,6 +38,12 @@
 -define(REQUEST_PAUSE_UPTO, 3).
 -define(CLIENT_MOD, rhc).
 
+-if(?OTP_RELEASE > 23).
+-define(RPC_MODULE, erpc).
+-else.
+-define(RPC_MODULE, rpc).
+-endif.
+
 -define(CONF,
  [{riak_kv,
  [
@@ -139,7 +145,7 @@ profile(Node) ->
  complete ->
  ok
  after ?PROFILE_PAUSE ->
- erpc:call(Node, riak_kv_util, profile_riak, [?PROFILE_LENGTH]),
+ ?RPC_MODULE:call(Node, riak_kv_util, profile_riak, [?PROFILE_LENGTH]),
  profile(Node)
  end.
 

diff --git a/tests/nextgenrepl_api_perf.erl b/tests/nextgenrepl_api_perf.erl
@@ -31,13 +31,19 @@
 -define(QUERY_EVERY, 5000).
 -define(GET_EVERY, 1).
 -define(LOG_EVERY, 10000).
--define(KEY_COUNT, 150000).
+-define(KEY_COUNT, 50000).
 -define(OBJECT_SIZE_BYTES, 32768).
 -define(REQUEST_PAUSE_UPTO, 3).
--define(CLIENT_MOD, rhc).
+-define(CLIENT_MOD, riakc_pb_socket).
 -define(SNK_WORKERS, 16).
 -define(FIELD_LIST, ["bin1"]).
 
+-if(?OTP_RELEASE > 23).
+-define(RPC_MODULE, erpc).
+-else.
+-define(RPC_MODULE, rpc).
+-endif.
+
 -define(CONFIG(SrcQueueDefns, WireCompress),
  [{riak_kv,
  [
@@ -78,14 +84,24 @@ confirm() ->
  ]),
 
  lager:info("Discover Peer IP/ports and restart with peer config"),
- FoldToPeerConfigHTTP = 
- fun(Node, Acc) ->
- {http, {IP, Port}} =
- lists:keyfind(http, 1, rt:connection_info(Node)),
- Acc0 = case Acc of "" -> ""; _ -> Acc ++ "|" end,
- Acc0 ++ IP ++ ":" ++ integer_to_list(Port) ++ ":http"
+ FoldToPeerConfig =
+ case ?CLIENT_MOD of
+ rhc ->
+ fun(Node, Acc) ->
+ {http, {IP, Port}} =
+ lists:keyfind(http, 1, rt:connection_info(Node)),
+ Acc0 = case Acc of "" -> ""; _ -> Acc ++ "|" end,
+ Acc0 ++ IP ++ ":" ++ integer_to_list(Port) ++ ":http"
+ end;
+ riakc_pb_socket ->
+ fun(Node, Acc) ->
+ {pb, {IP, Port}} =
+ lists:keyfind(pb, 1, rt:connection_info(Node)),
+ Acc0 = case Acc of "" -> ""; _ -> Acc ++ "|" end,
+ Acc0 ++ IP ++ ":" ++ integer_to_list(Port) ++ ":pb"
+ end
  end,
- reset_peer_config(FoldToPeerConfigHTTP, ClusterA, ClusterB),
+ reset_peer_config(FoldToPeerConfig, ClusterA, ClusterB),
 
  perf_test(hd(ClusterA), ?CLIENT_MOD, ?CLIENT_COUNT),
 
@@ -109,8 +125,32 @@ reset_peer_config(FoldToPeerConfig, ClusterA, ClusterB) ->
  ).
 
 get_memory(Node) ->
- MemStats = rpc:call(Node, erlang, memory, []),
- lager:info("MemStats for Node ~w ~p", [Node, MemStats]).
+ MemStats = ?RPC_MODULE:call(Node, erlang, memory, []),
+ lager:info("MemStats for Node ~w ~p", [Node, MemStats]),
+ Top10 = ?RPC_MODULE:call(Node, riak_kv_util, top_n_binary_total_memory, [10]),
+ lager:info("Top 10 processes for binary memory:"),
+ lists:foreach(
+ fun(BinProc) -> lager:info("Process memory stat ~p", [BinProc]) end,
+ Top10
+ ),
+ lager:info("Top 50 processes for binary - summary by initial call"),
+ Top50Summary =
+ ?RPC_MODULE:call(
+ Node, riak_kv_util, summarise_binary_memory_by_initial_call, [50]
+ ),
+ lists:foreach(fun(ICSumm) -> lager:info("~p", [ICSumm]) end, Top50Summary),
+ Top5 = ?RPC_MODULE:call(Node, riak_kv_util, top_n_process_total_memory, [5]),
+ lager:info("Top 5 processes for process memory:"),
+ lists:foreach(
+ fun(BinProc) -> lager:info("Process memory stat ~p", [BinProc]) end,
+ Top5
+ ),
+ lager:info("Top 100 processes for process memory - summary by initial call"),
+ Top100Summary =
+ ?RPC_MODULE:call(
+ Node, riak_kv_util, summarise_process_memory_by_initial_call, [100]
+ ),
+ lists:foreach(fun(ICSumm) -> lager:info("~p", [ICSumm]) end, Top100Summary).
 
 perf_test(Node, ClientMod, ClientCount) ->
  Clients = get_clients(ClientCount, Node, ClientMod),
@@ -140,8 +180,12 @@ perf_test(Node, ClientMod, ClientCount) ->
  SpawnFuns = lists:map(SpawnUpdateFun, ClientBPairs),
  lists:foreach(fun spawn/1, SpawnFuns),
 
+ Profiler = general_api_perf:spawn_profile_fun(Node),
+
  ok = receive_complete(0, length(Clients)),
 
+ Profiler ! complete,
+
  close_clients(Clients, ClientMod),
 
  EndTime = os:system_time(millisecond),
@@ -274,4 +318,3 @@ act(Client, ClientMod, Bucket, I, V) ->
  end,
  timer:sleep(rand:uniform(?REQUEST_PAUSE_UPTO))
  .
-