Skip to content

Commit

Permalink
Add JSON Object Value Generator
Browse files Browse the repository at this point in the history
Add `json_obj_valgen` to the Yokozuna BB driver. I used this to test
indexing "large" JSON objects while investigating #358.  The config
line to use this value generator looks like so:

```
{value_generator, {function, yz_driver, json_obj_valgen,
[<NumFields>]}}.
```

This will generate a static JSON object (i.e. generated once at start
up) which contains `<NumFields>` integer fields.  All workers will
index the identical object.  Here is an example config from one of my
testing runs:

```
{mode, max}.
{concurrent, 32}.
{driver, yz_driver}.
{code_paths, ["/root/work/yz-bb/misc/bench"]}.
{secure, false}.
{bucket, {<<"data">>, <<"largeobj">>}}.
{index, <<"largeobj">>}.
{pb_conns, [{"10.0.1.201", 8087}]}.
{http_conns, []}.
{duration, infinity}.
{key_generator, {to_binstr, "~B", {partitioned_sequential_int, 0,
1000}}}.
{value_generator, {function, yz_driver, json_obj_valgen, [4000]}}.
{operations, [{load_pb, 1}]}.
```

This uses 32 workers to write the same JSON object containing 4000
integer fields over the key space 0 to 999.
  • Loading branch information
rzezeski committed Apr 23, 2014
1 parent 9f87dc4 commit cc6d04f
Showing 1 changed file with 30 additions and 0 deletions.
30 changes: 30 additions & 0 deletions misc/bench/src/yz_driver.erl
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,19 @@ run(load_fruit_pb, KeyValGen, _, S=#state{bucket=Bucket, pb_conns=Conns}) ->
Err -> {error, Err, S2}
end;

run(load_pb, KeyGen, ValGen, S) ->
Bucket = S#state.bucket,
Conns = S#state.pb_conns,
Conn = get_conn(Conns),
Key = KeyGen(),
{CT, Val} = ValGen(),
Obj = riakc_obj:new(Bucket, Key, Val, CT),
S2 = S#state{pb_conns=wrap(Conns)},
case riakc_pb_socket:put(Conn, Obj, [{timeout, 90000}], 90000) of
ok -> {ok, S2};
Err -> {error, Err, S2}
end;

run({random_fruit_search_pb, FL, MaxTerms, MaxCardinality}, K, V, S=#state{fruits=undefined}) ->
S2 = S#state{fruits=gen_fruits(MaxCardinality)},
run({random_fruit_search_pb, FL, MaxTerms, MaxCardinality}, K, V, S2);
Expand Down Expand Up @@ -296,6 +309,23 @@ fruit_key_val_gen(Id, NumKeys) ->
always(_Id, Val) ->
fun() -> Val end.

%% @doc Generate a value generator function that returns a static JSON
%% object with `NumFields'. Each field has the name `num_<N>_i' where
%% N is the number of the field. The value for the field is N.
-spec json_obj_valgen(term(), pos_integer()) ->
fun(() -> {CT :: string(), JSON :: binary()}).
json_obj_valgen(_Id, NumFields) when NumFields > 0 ->
GenFieldVal =
fun(I) ->
Ib = ?INT_TO_BIN(I),
{<<"num_",Ib/binary,"_i">>, I}
end,
Obj = {struct, [GenFieldVal(I) || I <- lists:seq(1, NumFields)]},
JSON = iolist_to_binary(mochijson2:encode(Obj)),
fun() ->
{"application/json", JSON}
end.

%% ====================================================================
%% Private
%% ====================================================================
Expand Down

0 comments on commit cc6d04f

Please sign in to comment.