Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Soften JVM overload failures #358

Merged
merged 5 commits into from
Apr 22, 2014
Merged

Soften JVM overload failures #358

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c147db7
Soften jvm overload failures
coderoshi Apr 9, 2014
894e70d
Set maxWarmingSearchers back to default
coderoshi Apr 11, 2014
0c5736a
Change catch-all to ignored
coderoshi Apr 16, 2014
e823d0d
Merge remote-tracking branch 'origin/develop' into er/throughput
rzezeski Apr 22, 2014
61d6326
Use actual reserved word for test
rzezeski Apr 22, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions priv/conf/solrconfig.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
If both ramBufferSizeMB and maxBufferedDocs is set, then
Lucene will flush based on whichever limit is hit first. -->
<!-- Reduce disk flushes. If YZ loses data on crash, AAE will repair anyway -->
<ramBufferSizeMB>32</ramBufferSizeMB>
<!-- <ramBufferSizeMB>100</ramBufferSizeMB> -->

<!-- Expert: Merge Policy
The Merge Policy in Lucene controls how merging of segments is done.
Expand Down Expand Up @@ -324,7 +324,7 @@

<fieldValueCache class="solr.FastLRUCache"
size="512"
autowarmCount="128"
autowarmCount="0"
showItems="32" />


Expand Down Expand Up @@ -387,7 +387,7 @@
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
<maxWarmingSearchers>4</maxWarmingSearchers>

</query>

Expand Down
2 changes: 1 addition & 1 deletion priv/default_schema.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@
<dynamicField name="*_set" type="string" indexed="true" stored="false" multiValued="true" />

<!-- catch-all field -->
<dynamicField name="*" type="text_general" indexed="true" stored="false" multiValued="true" />
<dynamicField name="*" type="text_general" indexed="false" stored="false" multiValued="true" />
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we are not going to index the catch-all field then set the type to ignored and drop the other attributes so they are inherited from the type. IMO this makes the drop semantic obvious. It also may prevent unnecessary analyzing since it uses the StrField type (I would hope indexed=false prevents analysis but use a non-analyzed type just in case).


<field name="_yz_id" type="_yz_str" indexed="true" stored="true" multiValued="false" required="true"/>

Expand Down
7 changes: 5 additions & 2 deletions priv/solr/etc/jetty.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,11 @@
<Call class="java.lang.System" name="setProperty"> <Arg>log4j.configuration</Arg> <Arg>etc/log4j.properties</Arg> </Call>
<Set name="host"><SystemProperty name="jetty.host" /></Set>
<Set name="port"><SystemProperty name="jetty.port" default="8983"/></Set>
<Set name="maxIdleTime">50000</Set>
<Set name="lowResourceMaxIdleTime">1500</Set>
<Set name="maxIdleTime">60000</Set>
<!-- this could prematurely close sockets, but is better than tying up resources -->
<Set name="lowResourceMaxIdleTime">5000</Set>
<!-- acceptors should equal number of CPUs * 2 -->
<!-- <Set name="acceptors">16</Set> -->
<Set name="statsOn">false</Set>
</New>
</Arg>
Expand Down
15 changes: 14 additions & 1 deletion riak_test/yz_languages.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ confirm() ->
confirm_body_search_encoding(Cluster),
confirm_language_field_type(Cluster),
confirm_tag_encoding(Cluster),
confirm_reserved_word_safety(Cluster),
pass.

select_random(List) ->
Expand Down Expand Up @@ -61,9 +62,12 @@ store_and_search(Cluster, Bucket, Index, CT, Body, Field, Term) ->
store_and_search(Cluster, Bucket, Index, Headers, CT, Body, Field, Term).

store_and_search(Cluster, Bucket, Index, Headers, CT, Body, Field, Term) ->
store_and_search(Cluster, Bucket, Index, "test", Headers, CT, Body, Field, Term).

store_and_search(Cluster, Bucket, Index, Key, Headers, CT, Body, Field, Term) ->
HP = select_random(host_entries(rt:connection_info(Cluster))),
create_index(Cluster, HP, Index),
URL = bucket_url(HP, Bucket, "test"),
URL = bucket_url(HP, Bucket, Key),
lager:info("Storing to bucket ~s", [URL]),
{ok, "204", _, _} = ibrowse:send_req(URL, Headers, put, Body),
%% Sleep for soft commit
Expand Down Expand Up @@ -97,3 +101,12 @@ confirm_tag_encoding(Cluster) ->
{"x-riak-meta-yz-tags", "x-riak-meta-arabic_s"},
{"x-riak-meta-arabic_s", <<"أقرأ"/utf8>>}],
store_and_search(Cluster, Bucket, Index, Headers, "text/plain", Body, "arabic_s", "أقرأ").

confirm_reserved_word_safety(Cluster) ->
Index = <<"reserved">>,
Bucket = {Index, <<"b">>},
lager:info("confirm_reserved_word_safety ~s", [Index]),
Body = <<"whatever">>,
Headers = [{"Content-Type", "text/plain"}],
RKey = "ON",
store_and_search(Cluster, Bucket, Index, RKey, Headers, "text/plain", Body, "text", "whatever").
17 changes: 9 additions & 8 deletions src/yz_solr.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
-define(DEFAULT_URL, "http://localhost:8983/solr").
-define(DEFAULT_VCLOCK_N, 1000).
-define(QUERY(Str), {struct, [{'query', Str}]}).
-define(SOLR_TIMEOUT, 60000).

%% @doc This module provides the interface for making calls to Solr.
%% All interaction with Solr should go through this API.
Expand Down Expand Up @@ -63,7 +64,7 @@ commit(Core) ->
URL = ?FMT("~s/~s/update?~s", [base_url(), Core, Encoded]),
Headers = [{content_type, "application/json"}],
Opts = [{response_format, binary}],
case ibrowse:send_req(URL, Headers, post, JSON, Opts) of
case ibrowse:send_req(URL, Headers, post, JSON, Opts, ?SOLR_TIMEOUT) of
{ok, "200", _, _} -> ok;
Err -> throw({"Failed to commit", Err})
end.
Expand All @@ -72,7 +73,7 @@ commit(Core) ->
-spec core(atom(), proplist()) -> {ok, list(), binary()} |
{error, term()}.
core(Action, Props) ->
core(Action, Props, 5000).
core(Action, Props, ?SOLR_TIMEOUT).

-spec core(atom(), proplist(), ms()) -> {ok, list(), binary()} |
{error, term()}.
Expand Down Expand Up @@ -110,7 +111,7 @@ delete(Index, Ops) ->
URL = ?FMT("~s/~s/update", [base_url(), Index]),
Headers = [{content_type, "application/json"}],
Opts = [{response_format, binary}],
case ibrowse:send_req(URL, Headers, post, JSON, Opts) of
case ibrowse:send_req(URL, Headers, post, JSON, Opts, ?SOLR_TIMEOUT) of
{ok, "200", _, _} -> ok;
Err -> {error, Err}
end.
Expand Down Expand Up @@ -169,7 +170,7 @@ index(Core, Docs, DelOps) ->
URL = ?FMT("~s/~s/update", [base_url(), Core]),
Headers = [{content_type, "application/json"}],
Opts = [{response_format, binary}],
case ibrowse:send_req(URL, Headers, post, JSON, Opts) of
case ibrowse:send_req(URL, Headers, post, JSON, Opts, ?SOLR_TIMEOUT) of
{ok, "200", _, _} -> ok;
Err -> throw({"Failed to index docs", Ops, Err})
end.
Expand Down Expand Up @@ -197,7 +198,7 @@ partition_list(Core) ->
Encoded = mochiweb_util:urlencode(Params),
URL = ?FMT("~s/~s/select?~s", [base_url(), Core, Encoded]),
Opts = [{response_format, binary}],
case ibrowse:send_req(URL, [], get, [], Opts) of
case ibrowse:send_req(URL, [], get, [], Opts, ?SOLR_TIMEOUT) of
{ok, "200", _, Resp} -> {ok, Resp};
Err -> {error, Err}
end.
Expand Down Expand Up @@ -240,7 +241,7 @@ search(Core, Headers, Params) ->
URL = ?FMT("~s/~s/select", [base_url(), Core]),
Headers2 = [{content_type, "application/x-www-form-urlencoded"}|Headers],
Opts = [{response_format, binary}],
case ibrowse:send_req(URL, Headers2, post, Body, Opts) of
case ibrowse:send_req(URL, Headers2, post, Body, Opts, ?SOLR_TIMEOUT) of
{ok, "200", RHeaders, Resp} -> {RHeaders, Resp};
{ok, CodeStr, _, Err} ->
{Code, _} = string:to_integer(CodeStr),
Expand Down Expand Up @@ -320,10 +321,10 @@ encode_commit() ->
%% @doc Encode a delete operation into a mochijson2 compatiable term.
-spec encode_delete(delete_op()) -> term().
encode_delete({key,Key}) ->
Query = ?YZ_RK_FIELD_S ++ ":" ++ ibrowse_lib:url_encode(binary_to_list(Key)),
Query = ?YZ_RK_FIELD_S ++ ":\"" ++ ibrowse_lib:url_encode(binary_to_list(Key)) ++ "\"",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we are breatking UTF-8 here? I don't think we have tests for indexing/searching/deleting UTF-8 keys. I'll write up a separate issue.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This solves the reserved word problem by using a phrase query but the more appropriate solution down the line is to use the raw or term query parsers.

http://wiki.apache.org/solr/QueryParser

?QUERY(list_to_binary(Query));
encode_delete({siblings,Key}) ->
Query = ?YZ_RK_FIELD_S ++ ":" ++ ibrowse_lib:url_encode(binary_to_list(Key)) ++ " AND " ++ ?YZ_VTAG_FIELD_S ++ ":[* TO *]",
Query = ?YZ_RK_FIELD_S ++ ":\"" ++ ibrowse_lib:url_encode(binary_to_list(Key)) ++ "\" AND " ++ ?YZ_VTAG_FIELD_S ++ ":[* TO *]",
?QUERY(list_to_binary(Query));
encode_delete({'query', Query}) ->
?QUERY(Query);
Expand Down