RDBMS discovery backend for CETS

.circleci/template.yml

@@ -591,7 +591,7 @@ jobs:
       preset:
         type: enum
         enum: [internal_mnesia, mysql_redis, odbc_mssql_mnesia, ldap_mnesia,
-               elasticsearch_and_cassandra_mnesia, pgsql_mnesia, internal_cets]
+               elasticsearch_and_cassandra_mnesia, pgsql_mnesia, pgsql_cets]
         description: Preset to run
         default: internal_mnesia
       db:
@@ -826,11 +826,11 @@ workflows:
             - otp_25_docker
           filters: *all_tags
       - big_tests_in_docker:
-          name: internal_cets_25
-          executor: otp_25_redis
+          name: pgsql_cets_25
+          executor: otp_25_pgsql_redis
           context: mongooseim-org
-          preset: internal_cets
-          db: "mnesia cets"
+          preset: pgsql_cets
+          db: "mnesia postgres cets"
           requires:
             - otp_25_docker
           filters: *all_tags

big_tests/default.spec

@@ -116,6 +116,7 @@
 {suites, "tests", dynamic_domains_SUITE}.
 {suites, "tests", local_iq_SUITE}.
 {suites, "tests", tcp_listener_SUITE}.
+{suites, "tests", cets_disco_SUITE}.
 
 {config, ["test.config"]}.
 {logdir, "ct_report"}.

big_tests/dynamic_domains.spec

@@ -158,6 +158,7 @@
 {suites, "tests", domain_removal_SUITE}.
 {suites, "tests", local_iq_SUITE}.
 {suites, "tests", tcp_listener_SUITE}.
+{suites, "tests", cets_disco_SUITE}.
 
 {config, ["dynamic_domains.config", "test.config"]}.
 

big_tests/test.config

@@ -237,13 +237,31 @@
       {outgoing_pools, "[outgoing_pools.redis.global_distrib]
   scope = \"global\"
   workers = 10"}]},
-    {internal_cets,
-     [{dbs, [redis]},
+    {pgsql_cets,
+     [{dbs, [redis, pgsql]},
       {sm_backend, "\"cets\""},
       {stream_management_backend, cets},
+      {auth_method, "rdbms"},
+      {internal_databases, "[internal_databases.cets]
+  cluster_name = \"{{cluster_name}}\""},
       {outgoing_pools, "[outgoing_pools.redis.global_distrib]
   scope = \"global\"
-  workers = 10"}]},
+  workers = 10
+[outgoing_pools.rdbms.default]
+  scope = \"global\"
+  workers = 5
+  connection.driver = \"pgsql\"
+  connection.host = \"localhost\"
+  connection.database = \"ejabberd\"
+  connection.username = \"ejabberd\"
+  connection.password = \"mongooseim_secret\"
+  connection.tls.required = true
+  connection.tls.cacertfile = \"priv/ssl/cacert.pem\"
+  connection.tls.server_name_indication.enabled = false"},
+      {service_domain_db, ""},
+      {mod_vcard, "  backend = \"rdbms\"
+  host = \"vjud.@HOST@\"\n"},
+      {mod_roster, "  backend = \"rdbms\"\n"}]},
     {pgsql_mnesia,
      [{dbs, [redis, pgsql]},
       {auth_method, "rdbms"},

big_tests/tests/cets_disco_SUITE.erl

@@ -0,0 +1,72 @@
+-module(cets_disco_SUITE).
+-compile([export_all, nowarn_export_all]).
+
+-import(distributed_helper, [mim/0, rpc/4]).
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Suite configuration
+%%--------------------------------------------------------------------
+
+all() ->
+    [{group, file}, {group, rdbms}].
+
+groups() ->
+    [{file, [], file_cases()},
+     {rdbms, [], rdbms_cases()}].
+
+file_cases() ->
+    [file_backend].
+
+rdbms_cases() ->
+    [rdbms_backend].
+
+suite() ->
+    escalus:suite().
+
+%%--------------------------------------------------------------------
+%% Init & teardown
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+    escalus:init_per_suite(Config).
+
+end_per_suite(Config) ->
+    escalus:end_per_suite(Config).
+
+init_per_group(rdbms, Config) ->
+    case not ct_helper:is_ct_running()
+         orelse mongoose_helper:is_rdbms_enabled(domain_helper:host_type()) of
+        false -> {skip, rdbms_or_ct_not_running};
+        true -> Config
+    end;
+init_per_group(_, Config) ->
+    Config.
+
+end_per_group(_, Config) ->
+    Config.
+
+init_per_testcase(CaseName, Config) ->
+    escalus:init_per_testcase(CaseName, Config).
+
+end_per_testcase(CaseName, Config) ->
+    escalus:end_per_testcase(CaseName, Config).
+
+%%--------------------------------------------------------------------
+%% Test cases
+%%--------------------------------------------------------------------
+
+file_backend(Config) ->
+    Path = filename:join(?config(mim_data_dir, Config), "nodes.txt"),
+    Opts = #{disco_file => Path},
+    State = rpc(mim(), cets_discovery_file, init, [Opts]),
+    {{ok, Nodes}, _} = rpc(mim(), cets_discovery_file, get_nodes, [State]),
+    ['node1@localhost', 'node2@otherhost'] = lists:sort(Nodes).
+
+rdbms_backend(_Config) ->
+    Opts1 = #{cluster_name => <<"big_test">>, node_name_to_insert => <<"test1">>},
+    Opts2 = #{cluster_name => <<"big_test">>, node_name_to_insert => <<"test2">>},
+    State1 = rpc(mim(), mongoose_cets_discovery_rdbms, init, [Opts1]),
+    rpc(mim(), mongoose_cets_discovery_rdbms, get_nodes, [State1]),
+    State2 = rpc(mim(), mongoose_cets_discovery_rdbms, init, [Opts2]),
+    {{ok, Nodes}, _} = rpc(mim(), mongoose_cets_discovery_rdbms, get_nodes, [State2]),
+    [test1, test2] = lists:sort(Nodes).

big_tests/tests/cets_disco_SUITE_data/nodes.txt

@@ -0,0 +1,2 @@
+node1@localhost
+node2@otherhost

doc/configuration/configuration-files.md

@@ -15,6 +15,7 @@ The file is divided into the following sections:
 * [**general**](general.md) - Served XMPP domains, log level, server language and some other miscellaneous settings.
 * [**listen**](listen.md) - Configured listeners, receiving incoming XMPP and HTTP connections.
 * [**auth**](auth.md) - Supported client authentication methods and their options.
+* [**internal_databases**](internal-databases.md) - Options for Mnesia and CETS. They are primarily used for clustering.
 * [**outgoing_pools**](outgoing-connections.md) - Outgoing connections to external services, including databases, message queues and HTTP services.
 * [**services**](Services.md) - Internal services like an administration API and system metrics.
 * [**modules**](Modules.md) - [XMPP extension](https://xmpp.org/extensions/) modules, which extend the basic functionality provided by XMPP.

doc/configuration/internal-databases.md

@@ -0,0 +1,67 @@
+Internal databases are used to cluster MongooseIM nodes, and to replicate session list data between them.
+
+Mnesia is a legacy way to cluster MongooseIM nodes. It is also could be used to store persistent data, but we recommend
+to use RDBMS databases instead because of scalability and stability reasons.
+
+CETS is a new way to cluster MongooseIM nodes.
+CETS needs to know a list of nodes for the node discovery. There are two ways to get a list of nodes:
+
+- A text file with a list of nodes on each line. It is useful when there is an external script to make this file based on
+  some custom logic (for example, a bash script that uses AWS CLI to discover instances in the autoscaling group). This file
+  would be automatilly reread on change.
+- RDBMS database. MongooseIM would write into RDBMS its nodename and read a list of other nodes. It is pretty simple, but
+  RDBMS database could be a single point of failure.
+
+Section example:
+
+```toml
+[internal_databases]
+    [internal_databases.mnesia]
+
+    [internal_databases.cets]
+        backend = "rdbms"
+        cluster_name = "mongooseim"
+```
+
+or
+
+```toml
+[internal_databases]
+    [internal_databases.cets]
+        backend = "file"
+        node_list_file = "cets_disco.txt"
+```
+
+To enable just CETS, define only `internal_databases.cets` section:
+
+```toml
+[internal_databases]
+    [internal_databases.cets]
+```
+
+# CETS Options
+
+### `internal_databases.cets.backend`
+
+Backend for CETS discovery.
+
+* **Syntax:** string, one of `"rdbms"`, `"file"`.
+* **Default:** `"rdbms"`
+* **Example:** `backend = "rdbms"`
+
+### `internal_databases.cets.cluster_name`
+
+Namespace for the cluster. Only nodes with the same cluster name would be discoverd. This option is for RDBMS backend.
+
+* **Syntax:** string.
+* **Default:** `"mongooseim"`
+* **Example:** `cluster_name = "mongooseim"`
+
+### `internal_databases.cets.node_list_file`
+
+File to read a list of nodes from. Relative to the MongooseIM's release directory. This option is for the file backend.
+Required, if `backend = "file"`.
+
+* **Syntax:** path.
+* **Default:** not specified.
+* **Example:** `node_list_file = "/etc/mim_nodes.txt"`

mkdocs.yml

@@ -69,6 +69,7 @@ nav:
       - 'Options: General': 'configuration/general.md'
       - 'Options: Listen': 'configuration/listen.md'
       - 'Options: Auth': 'configuration/auth.md'
+      - 'Options: Internal Databases': 'configuration/internal-databases.md'
       - 'Options: Outgoing connections': 'configuration/outgoing-connections.md'
       - 'Options: Services': 'configuration/Services.md'
       - 'Options: Extension Modules': 'configuration/Modules.md'

priv/mssql2012.sql

@@ -752,3 +752,10 @@ CREATE TABLE domain_events (
     domain VARCHAR(250) NOT NULL
 );
 CREATE INDEX i_domain_events_domain ON domain_events(domain);
+
+CREATE TABLE discovery_nodes (
+    node_name varchar(250),
+    cluster_name varchar(250),
+    updated_timestamp BIGINT NOT NULL, -- in microseconds
+    PRIMARY KEY (cluster_name, node_name)
+);

priv/mysql.sql

@@ -544,3 +544,10 @@ CREATE TABLE domain_events (
     domain VARCHAR(250) NOT NULL
 );
 CREATE INDEX i_domain_events_domain ON domain_events(domain);
+
+CREATE TABLE discovery_nodes (
+    node_name varchar(250),
+    cluster_name varchar(250),
+    updated_timestamp BIGINT NOT NULL, -- in microseconds
+    PRIMARY KEY (cluster_name, node_name)
+);

priv/pg.sql

@@ -504,3 +504,10 @@ CREATE TABLE domain_events (
     PRIMARY KEY(id)
 );
 CREATE INDEX i_domain_events_domain ON domain_events(domain);
+
+CREATE TABLE discovery_nodes (
+    node_name varchar(250),
+    cluster_name varchar(250),
+    updated_timestamp BIGINT NOT NULL, -- in microseconds
+    PRIMARY KEY (cluster_name, node_name)
+);

rel/fed1.vars-toml.config

@@ -17,6 +17,7 @@
 %% "localhost" host should NOT be defined.
 {hosts, "\"fed1\""}.
 {default_server_domain, "\"fed1\""}.
+{cluster_name, "fed"}.
 
 %% domain.example.com is for multitenancy preset, muc_SUITE:register_over_s2s
 {s2s_addr, "[[s2s.address]]

rel/files/mongooseim.toml

@@ -162,6 +162,8 @@
     {{{auth_method_opts}}}{{/auth_method_opts}}
   {{/auth_method}}
 
+{{{internal_databases}}}
+
 {{#outgoing_pools}}
 {{{outgoing_pools}}}
 {{/outgoing_pools}}

rel/mim1.vars-toml.config

@@ -20,6 +20,7 @@
 {hosts, "\"localhost\", \"anonymous.localhost\", \"localhost.bis\""}.
 {host_types, "\"test type\", \"dummy auth\", \"anonymous\""}.
 {default_server_domain, "\"localhost\""}.
+{cluster_name, "mim"}.
 
 {mod_amp, ""}.
 {host_config,

rel/mim2.vars-toml.config

@@ -18,6 +18,7 @@
 {hosts, "\"localhost\", \"anonymous.localhost\", \"localhost.bis\""}.
 {host_types, "\"test type\", \"dummy auth\""}.
 {default_server_domain, "\"localhost\""}.
+{cluster_name, "mim"}.
 {s2s_addr, "[[s2s.address]]
     host = \"localhost2\"
     ip_address = \"127.0.0.1\""}.

rel/mim3.vars-toml.config

@@ -20,6 +20,7 @@
 
 {hosts, "\"localhost\", \"anonymous.localhost\", \"localhost.bis\""}.
 {default_server_domain, "\"localhost\""}.
+{cluster_name, "mim"}.
 
 {s2s_addr, "[[s2s.address]]
     host = \"localhost2\"

rel/reg1.vars-toml.config

@@ -21,6 +21,8 @@
 %% "reg1" is a local host.
 {hosts, "\"reg1\", \"localhost\""}.
 {default_server_domain, "\"reg1\""}.
+{cluster_name, "reg"}.
+
 {s2s_addr, "[[s2s.address]]
     host = \"localhost\"
     ip_address = \"127.0.0.1\"

rel/vars-toml.config

@@ -17,6 +17,8 @@
 {http_api_client_endpoint, "port = {{ http_api_client_endpoint_port }}"}.
 {s2s_use_starttls, "\"optional\""}.
 {s2s_certfile, "\"priv/ssl/fake_server.pem\""}.
+{internal_databases, "[internal_databases]
+    [internal_databases.mnesia]"}.
 
 "./configure.vars.config".
 

src/config/mongoose_config_spec.erl

@@ -91,6 +91,7 @@ root() ->
                  <<"listen">> => Listen#section{include = always},
                  <<"auth">> => Auth#section{include = always},
                  <<"outgoing_pools">> => outgoing_pools(),
+                 <<"internal_databases">> => internal_databases(),
                  <<"services">> => services(),
                  <<"modules">> => Modules#section{include = always},
                  <<"shaper">> => shaper(),
@@ -425,6 +426,32 @@ auth_password() ->
        include = always
       }.
 
+%% path: internal_databases
+internal_databases() ->
+    Items = #{<<"cets">> => internal_database_cets(),
+              <<"mnesia">> => internal_database_mnesia()},
+    #section{items = Items,
+             format_items = map,
+             wrap = global_config,
+             include = always}.
+
+%% path: internal_databases.*.*
+internal_database_cets() ->
+    #section{
+       items = #{<<"backend">> => #option{type = atom,
+                                          validate = {enum, [file, rdbms]}},
+                 <<"cluster_name">> => #option{type = atom, validate = non_empty},
+                 %% Relative to the release directory (or an absolute name)
+                 <<"node_list_file">> => #option{type = string,
+                                                 validate = filename}
+                },
+       defaults = #{<<"backend">> => rdbms, <<"cluster_name">> => mongooseim}
+      }.
+
+%% path: internal_databases.*.*
+internal_database_mnesia() ->
+    #section{}.
+
 %% path: outgoing_pools
 outgoing_pools() ->
     PoolTypes = [<<"cassandra">>, <<"elastic">>, <<"http">>, <<"ldap">>,