From fc6729ca4713c5015b587c9461e2a4327f45b37a Mon Sep 17 00:00:00 2001 From: Michal Piotrowski Date: Fri, 22 May 2015 15:14:43 +0200 Subject: [PATCH 1/6] add mapred function to mongoose_riak --- apps/ejabberd/src/mongoose_riak.erl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/apps/ejabberd/src/mongoose_riak.erl b/apps/ejabberd/src/mongoose_riak.erl index 83d88bac1e1..bc5a5af95e4 100644 --- a/apps/ejabberd/src/mongoose_riak.erl +++ b/apps/ejabberd/src/mongoose_riak.erl @@ -30,9 +30,11 @@ -export([update_type/3, update_type/4]). -export([fetch_type/2, fetch_type/3]). -export([list_keys/1]). +-export([list_buckets/1]). -export([get_worker/0]). -export([create_new_map/1]). -export([update_map/2]). +-export([mapred/2]). -export([pool_name/0]). @@ -121,6 +123,9 @@ fetch_type(Bucket, Key, Opts) -> list_keys(Bucket) -> ?CALL(list_keys, [Bucket]). +-spec list_buckets(binary()) -> list(). +list_buckets(Type) -> + ?CALL(list_buckets, [Type]). -spec create_new_map([riakc_map_op()]) -> riakc_map:crdt_map(). create_new_map(Ops) -> @@ -130,6 +135,11 @@ create_new_map(Ops) -> update_map(Map, Ops) -> lists:foldl(fun update_map_op/2, Map, Ops). +-spec mapred(mapred_inputs(), [mapred_queryterm()]) -> + {ok, mapred_result()} | {error, term()}. +mapred(KeyFileters, MapRed) -> + ?CALL(mapred, [KeyFileters, MapRed]). + -spec get_worker() -> pid() | undefined. get_worker() -> case catch cuesport:get_worker(pool_name()) of From 8016e7385256af52a72c1bb870b7cbf76ca323fa Mon Sep 17 00:00:00 2001 From: Michal Piotrowski Date: Fri, 22 May 2015 15:15:16 +0200 Subject: [PATCH 2/6] add riak backend for private storage --- apps/ejabberd/src/mod_private_riak.erl | 75 ++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 apps/ejabberd/src/mod_private_riak.erl diff --git a/apps/ejabberd/src/mod_private_riak.erl b/apps/ejabberd/src/mod_private_riak.erl new file mode 100644 index 00000000000..1120fd12bb3 --- /dev/null +++ b/apps/ejabberd/src/mod_private_riak.erl @@ -0,0 +1,75 @@ +%%============================================================================== +%% Copyright 2015 Erlang Solutions Ltd. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%============================================================================== +-module(mod_private_riak). + +-behaviour(mod_private). + +%% API +-export([init/2, + multi_set_data/3, + multi_get_data/3, + remove_user/2]). + +-include("ejabberd.hrl"). +-include("jlib.hrl"). + +-spec init(ejabberd:lserver(), list()) -> ok. +init(_Host, _Opts) -> + ok. + +-spec multi_set_data(ejabberd:luser(), ejabberd:lserver(), + {binary(), ejabberd:xmlterm()}) -> ok. +multi_set_data(LUser, LServer, NS2XML) -> + [set_private_data(LUser, LServer, NS, XML) || {NS, XML} <- NS2XML], + ok. + +-spec multi_get_data(ejabberd:luser(), ejabberd:lserver(), + {binary(), term()}) -> ok. +multi_get_data(LUser, LServer, NS2Def) -> + [get_private_data(LUser, LServer, NS, Default) || {NS, Default} <- NS2Def]. + +-spec remove_user(ejabberd:luser(), ejabberd:lserver()) -> ok. +remove_user(LUser, LServer) -> + KeyFilter = [[<<"starts_with">>,LUser]], + Bucket = bucket_type(LServer), + case mongoose_riak:mapred({Bucket, KeyFilter}, []) of + {ok, []} -> + ok; + {ok, [{0, BucketKeys} | _]} -> + [mongoose_riak:delete(Bucket1, Key) || {{Bucket1, Key}, _} <- BucketKeys]; + Error -> + ?WARNING_MSG("Error reading keys to remove: ~p", [Error]), + {error, Error} + end. + +set_private_data(LUser, LServer, NS, XML) -> + Obj = riakc_obj:new(bucket_type(LServer), key(LUser, NS), xml:element_to_binary(XML)), + ok = mongoose_riak:put(Obj). + +get_private_data(LUser, LServer, NS, Default) -> + case mongoose_riak:get(bucket_type(LServer), key(LUser, NS)) of + {ok, Obj} -> + Value = riakc_obj:get_value(Obj), + #xmlel{} = xml_stream:parse_element(Value); + _ -> + Default + end. + +bucket_type(LServer) -> + {<<"private">>, LServer}. + +key(LUser, NS) -> + <>. \ No newline at end of file From 4b6a7081d83c00727eb18fe36b2aca5188ac6b96 Mon Sep 17 00:00:00 2001 From: Michal Piotrowski Date: Fri, 22 May 2015 15:15:37 +0200 Subject: [PATCH 3/6] change returned value for multi_set private function --- apps/ejabberd/src/mod_private.erl | 4 ++-- apps/ejabberd/src/mod_private_mnesia.erl | 3 ++- apps/ejabberd/src/mod_private_mysql.erl | 2 +- apps/ejabberd/src/mod_private_odbc.erl | 9 +++++---- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/apps/ejabberd/src/mod_private.erl b/apps/ejabberd/src/mod_private.erl index f09f1cacb79..9f0895cf9a9 100644 --- a/apps/ejabberd/src/mod_private.erl +++ b/apps/ejabberd/src/mod_private.erl @@ -53,7 +53,7 @@ NS :: binary(), XML :: #xmlel{}, Reason :: term(), - Result :: {atomic, ok} | {aborted, Reason} | {error, Reason}. + Result :: ok | {aborted, Reason} | {error, Reason}. -callback multi_get_data(LUser, LServer, NS2Def) -> [XML | Default] when LUser :: binary(), @@ -109,7 +109,7 @@ process_sm_iq( NS2XML = to_map(Elems), Result = ?BACKEND:multi_set_data(LUser, LServer, NS2XML), case Result of - {atomic, ok} -> + ok -> IQ#iq{type = result, sub_el = [SubElem]}; {error, Reason} -> ?ERROR_MSG("~p:multi_set_data failed ~p for ~ts@~ts.", diff --git a/apps/ejabberd/src/mod_private_mnesia.erl b/apps/ejabberd/src/mod_private_mnesia.erl index 55cb69211a1..cc5137e4bd4 100644 --- a/apps/ejabberd/src/mod_private_mnesia.erl +++ b/apps/ejabberd/src/mod_private_mnesia.erl @@ -51,7 +51,8 @@ init(_Host, _Opts) -> multi_set_data(LUser, LServer, NS2XML) -> F = fun() -> multi_set_data_t(LUser, LServer, NS2XML) end, - mnesia:transaction(F). + {atomic, ok} = mnesia:transaction(F), + ok. multi_set_data_t(LUser, LServer, NS2XML) -> [set_data_t(LUser, LServer, NS, XML) || {NS, XML} <- NS2XML], diff --git a/apps/ejabberd/src/mod_private_mysql.erl b/apps/ejabberd/src/mod_private_mysql.erl index a95d6fd9e99..80f920c9397 100644 --- a/apps/ejabberd/src/mod_private_mysql.erl +++ b/apps/ejabberd/src/mod_private_mysql.erl @@ -21,7 +21,7 @@ multi_set_data(LUser, LServer, NS2XML) -> replace_like_insert_result( odbc_queries:multi_set_private_data(LServer, SLUser, Rows)). -replace_like_insert_result({updated, _}) -> {atomic, ok}; +replace_like_insert_result({updated, _}) -> ok; replace_like_insert_result({error, Reason}) -> {error, Reason}; replace_like_insert_result({aborted, Reason}) -> {aborted, Reason}. diff --git a/apps/ejabberd/src/mod_private_odbc.erl b/apps/ejabberd/src/mod_private_odbc.erl index b6a14d829d8..bd6b9a0a9e0 100644 --- a/apps/ejabberd/src/mod_private_odbc.erl +++ b/apps/ejabberd/src/mod_private_odbc.erl @@ -44,14 +44,15 @@ init(_Host, _Opts) -> multi_set_data(LUser, LServer, NS2XML) -> F = fun() -> multi_set_data_t(LUser, LServer, NS2XML) end, - odbc_queries:sql_transaction(LServer, F). + {atomic, ok} = odbc_queries:sql_transaction(LServer, F), + ok. multi_set_data_t(LUser, LServer, NS2XML) -> - [set_data_t(LUser, LServer, NS, XML) || {NS, XML} <- NS2XML], + SLUser = ejabberd_odbc:escape(LUser), + [set_data_t(SLUser, LServer, NS, XML) || {NS, XML} <- NS2XML], ok. -set_data_t(LUser, LServer, NS, XML) -> - SLUser = ejabberd_odbc:escape(LUser), +set_data_t(SLUser, LServer, NS, XML) -> SNS = ejabberd_odbc:escape(NS), SData = ejabberd_odbc:escape(xml:element_to_binary(XML)), odbc_queries:set_private_data(LServer, SLUser, SNS, SData). From b79a57b480c0635ed77c501aa043612c97943a37 Mon Sep 17 00:00:00 2001 From: Michal Piotrowski Date: Tue, 30 Jun 2015 11:28:06 +0200 Subject: [PATCH 4/6] create private bucket type --- tools/setup_riak | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/setup_riak b/tools/setup_riak index fe4dfff8e09..07ab78ecb67 100755 --- a/tools/setup_riak +++ b/tools/setup_riak @@ -6,3 +6,6 @@ service riak start ${1}riak-admin bucket-type create users '{"props":{"datatype":"map"}}' ${1}riak-admin bucket-type activate users + +${1}riak-admin bucket-type create private '{"props":{"last_write_wins":true}}' +${1}riak-admin bucket-type activate private From d8de5481d73ecc86f05e6ea8a79a69a6a82f30b3 Mon Sep 17 00:00:00 2001 From: Pawel Pikula Date: Tue, 18 Aug 2015 11:25:50 +0200 Subject: [PATCH 5/6] update docs --- doc/advanced-configuration/database-backends-configuration.md | 4 ++++ doc/modules/mod_private.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/advanced-configuration/database-backends-configuration.md b/doc/advanced-configuration/database-backends-configuration.md index 217bd136935..fcc031fbc57 100644 --- a/doc/advanced-configuration/database-backends-configuration.md +++ b/doc/advanced-configuration/database-backends-configuration.md @@ -71,6 +71,7 @@ psql -h localhost -U user -q -d mongooseim -f pg.sql **Can be used for:** * users (credentials) +* private storage **Setup** @@ -80,6 +81,9 @@ be able to store **user credentials** one have to run the following command: ```bash riak-admin bucket-type create users '{"props":{"datatype":"map"}}' riak-admin bucket-type activate users + +riak-admin bucket-type create private '{"props":{"last_write_wins":true}}' +riak-admin bucket-type activate private ``` This will create a bucket type required for storing **users credentials** and it will diff --git a/doc/modules/mod_private.md b/doc/modules/mod_private.md index 3c9c7e1aace..6e02a7916f6 100644 --- a/doc/modules/mod_private.md +++ b/doc/modules/mod_private.md @@ -3,7 +3,7 @@ This module implements [XEP-0049 (Private XML Storage)](http://xmpp.org/extensio ### Options * **iqdisc** -* **backend** (atom, default: `mnesia`) - Storage backend. Currently `mnesia`, `odbc` and `mysql` are supported . `mysql` uses MySQL-specific queries so in some cases it is more efficient than generic `odbc`. +* **backend** (atom, default: `mnesia`) - Storage backend. Currently `mnesia`, `odbc`, `riak` and `mysql` are supported . `mysql` uses MySQL-specific queries so in some cases it is more efficient than generic `odbc`. ### Example Configuration ``` From f247165f541962b049fa33ee8ef65535d9d8a1cc Mon Sep 17 00:00:00 2001 From: Pawel Pikula Date: Wed, 19 Aug 2015 09:37:58 +0200 Subject: [PATCH 6/6] Unify behaviour and return value in private backends Previous commit introduced badmatch, which I believe wasn't properly handled by the layer above. --- apps/ejabberd/src/mod_private_mnesia.erl | 6 ++++-- apps/ejabberd/src/mod_private_odbc.erl | 7 +++++-- apps/ejabberd/src/mod_private_riak.erl | 10 +++++++--- doc/modules/mod_private.md | 4 ++++ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/apps/ejabberd/src/mod_private_mnesia.erl b/apps/ejabberd/src/mod_private_mnesia.erl index cc5137e4bd4..3c04e898843 100644 --- a/apps/ejabberd/src/mod_private_mnesia.erl +++ b/apps/ejabberd/src/mod_private_mnesia.erl @@ -51,8 +51,10 @@ init(_Host, _Opts) -> multi_set_data(LUser, LServer, NS2XML) -> F = fun() -> multi_set_data_t(LUser, LServer, NS2XML) end, - {atomic, ok} = mnesia:transaction(F), - ok. + case mnesia:transaction(F) of + {atomic, ok} -> ok; + {aborted, Reason} -> {aborted, Reason} + end. multi_set_data_t(LUser, LServer, NS2XML) -> [set_data_t(LUser, LServer, NS, XML) || {NS, XML} <- NS2XML], diff --git a/apps/ejabberd/src/mod_private_odbc.erl b/apps/ejabberd/src/mod_private_odbc.erl index bd6b9a0a9e0..d732d1cffc7 100644 --- a/apps/ejabberd/src/mod_private_odbc.erl +++ b/apps/ejabberd/src/mod_private_odbc.erl @@ -44,8 +44,11 @@ init(_Host, _Opts) -> multi_set_data(LUser, LServer, NS2XML) -> F = fun() -> multi_set_data_t(LUser, LServer, NS2XML) end, - {atomic, ok} = odbc_queries:sql_transaction(LServer, F), - ok. + case odbc_queries:sql_transaction(LServer, F) of + {atomic, ok} -> ok; + {aborted, Reason} -> {aborted, Reason}; + {error, Reason} -> {error, Reason} + end. multi_set_data_t(LUser, LServer, NS2XML) -> SLUser = ejabberd_odbc:escape(LUser), diff --git a/apps/ejabberd/src/mod_private_riak.erl b/apps/ejabberd/src/mod_private_riak.erl index 1120fd12bb3..b671e7d1771 100644 --- a/apps/ejabberd/src/mod_private_riak.erl +++ b/apps/ejabberd/src/mod_private_riak.erl @@ -33,8 +33,12 @@ init(_Host, _Opts) -> -spec multi_set_data(ejabberd:luser(), ejabberd:lserver(), {binary(), ejabberd:xmlterm()}) -> ok. multi_set_data(LUser, LServer, NS2XML) -> - [set_private_data(LUser, LServer, NS, XML) || {NS, XML} <- NS2XML], - ok. + R = [set_private_data(LUser, LServer, NS, XML) || {NS, XML} <- NS2XML], + %% check if something returned with error msg + case lists:keyfind(error, 1, R) of + {error, Reason} -> {error, Reason}; + false -> ok + end. -spec multi_get_data(ejabberd:luser(), ejabberd:lserver(), {binary(), term()}) -> ok. @@ -57,7 +61,7 @@ remove_user(LUser, LServer) -> set_private_data(LUser, LServer, NS, XML) -> Obj = riakc_obj:new(bucket_type(LServer), key(LUser, NS), xml:element_to_binary(XML)), - ok = mongoose_riak:put(Obj). + mongoose_riak:put(Obj). get_private_data(LUser, LServer, NS, Default) -> case mongoose_riak:get(bucket_type(LServer), key(LUser, NS)) of diff --git a/doc/modules/mod_private.md b/doc/modules/mod_private.md index 6e02a7916f6..1ff79f762c0 100644 --- a/doc/modules/mod_private.md +++ b/doc/modules/mod_private.md @@ -5,6 +5,10 @@ This module implements [XEP-0049 (Private XML Storage)](http://xmpp.org/extensio * **iqdisc** * **backend** (atom, default: `mnesia`) - Storage backend. Currently `mnesia`, `odbc`, `riak` and `mysql` are supported . `mysql` uses MySQL-specific queries so in some cases it is more efficient than generic `odbc`. +**CAUTION:** Riak backend doesn't support transactions(rollbacks), so please avoid inserting more +than one value in one set request, otherwise you may end up with partially save data, backend returns +first error. + ### Example Configuration ``` {mod_private, []}