test: add a few more tests

README.md

@@ -109,6 +109,12 @@ After convertion, the output tarball contains a structure similar to this:
   {atomic, [emqx_authn_mnesia]}
   ```
 
+- If you have retained messages being imported (from `mnesia/emqx_retainer_message`), then, after importing that table with the command above, you must run:
+
+  ```sh
+  emqx ctl retainer reindex start
+  ```
+
 # Build
 
 ## Prerequisites
@@ -155,6 +161,28 @@ rebar3 escriptize
 _build/default/bin/emqx_data_converter --data-files-dir /input /input/emqx-export-2023-7-13-15-52-15.json
 ```
 
+## Running tests
+
+To run the test suite locally, first compile the escript and copy it to the project root.
+
+```sh
+rebar3 escriptize
+cp _build/default/bin/emqx_data_converter ./
+```
+
+Then, run the script:
+
+```sh
+source test/scripts/env.sh
+test/scripts/test-convert-and-load.exs
+```
+
+If you want to run only tests with a specific tag, for example, only those tagged with `bridges` (i.e., those that have `@tag :bridges` above them):
+
+```sh
+test/scripts/test-convert-and-load.exs --only bridges
+```
+
 # Bundle the `escript` with Erlang/OTP
 
 ```shell

src/emqx_data_converter.erl

@@ -1487,8 +1487,8 @@ with_common_connnector_fields(ResParams, ConnConf) ->
 
 do_convert_action_resource(?DATA_ACTION, ActId, Args, ResId,
                           <<"backend_redis_", RedisType/binary>>, ResConf) ->
-    #{<<"cmd">> := Cmd} = Args,
-    redis_bridge(ActId, Cmd, ResId, RedisType, ResConf);
+    #{<<"cmd">> := _Cmd} = Args,
+    redis_action_resource(ActId, Args, ResId, RedisType, ResConf);
 do_convert_action_resource(?DATA_ACTION, ActId, Args, ResId, <<"backend_", RDBMS/binary>>, ResConf)
   when RDBMS =:= <<"pgsql">>;
        RDBMS =:= <<"mysql">>;
@@ -1592,18 +1592,30 @@ common_args_to_res_opts(Args) ->
             ResOpts
     end.
 
-redis_bridge(ActionId, Cmd, ResId, RedisType, ResConf) ->
-    BridgeName = bridge_name(ResId, ActionId),
+redis_action_resource(ActionId, #{<<"cmd">> := Cmd} = Args, ResId, RedisType, ResConf) ->
     CommonFields = [<<"server">>, <<"servers">>, <<"pool_size">>,
                     <<"database">>, <<"password">>, <<"sentinel">>],
-    OutConf = filter_out_empty(maps:with(CommonFields, ResConf)),
-    OutConf1 = case RedisType of
-                   <<"cluster">> -> maps:remove(<<"database">>, OutConf);
-                   _  -> OutConf
-               end,
-    OutConf2 = OutConf1#{<<"command_template">> => [bin(L) || L <- string:lexemes(str(Cmd), " ")],
-                         <<"ssl">> => convert_ssl_opts(maps:get(<<"ssl">>, ResConf, false), ResConf)},
-    {<<"redis_", RedisType/binary>>, BridgeName, OutConf2}.
+    ConnParams0 = filter_out_empty(maps:with(CommonFields, ResConf)),
+    ConnParams1 = case RedisType of
+        <<"cluster">> -> maps:remove(<<"database">>, ConnParams0);
+        _  -> ConnParams0
+    end,
+    ConnParams = ConnParams1#{<<"redis_type">> => RedisType},
+    ConnectorConf = #{
+        <<"parameters">> => ConnParams,
+        <<"ssl">> => convert_ssl_opts(maps:get(<<"ssl">>, ResConf, false), ResConf)
+    },
+    ActionParams =
+        #{ <<"redis_type">> => RedisType
+         , <<"command_template">> => [bin(L) || L <- string:lexemes(str(Cmd), " ")]
+         },
+    ActionConf = #{
+        <<"parameters">> => ActionParams,
+        <<"resource_opts">> => common_args_to_res_opts(Args)
+    },
+    Action = {<<"redis">>, make_action_name(ResId), ActionConf},
+    Connector = {<<"redis">>, make_connector_name(ResId), ConnectorConf},
+    {Action, Connector}.
 
 sqldb_action_resource(RDBMS, _ActionId, #{<<"sql">> := SQL} = Args, ResId, ResConf) ->
     ResConf1 = case ResConf of

test/data/auth-builtin-redis-postgres1.json

@@ -0,0 +1 @@
+{"version":"4.4","rules":[],"resources":[],"blacklist":[{"who":{"username":"banned_username2"},"by":"user","reason":"banned2","at":1728916029,"until":1729134000}],"apps":[{"id":"admin","secret":"public","name":"Default","desc":"Application user","status":true,"expired":"undefined"},{"id":"app_id","secret":"4mVZvVT9CnC6Z3AYdk9C07Ecz9AuBCLblb43kk69BcxbBhP","name":"app_name","desc":"app comment","status":true,"expired":"undefined"}],"users":[{"username":"admin","password":"kY+vJrDS/DCvsAPMYG09iAWLdKY=","tags":{"role":"administrator","desc":"admin"}},{"username":"dash_user","password":"G9wNtibQhQoBqfbfuXGADBhZQgE=","tags":{"role":"administrator","desc":"dash comment"}}],"auth_mnesia":[{"login":"authn_cid1","type":"clientid","password":"6v6dezYyY2EzNGQzYWYyZGQ1NjQyYjg4ZjRlMzQ3OGYzY2IzZDU4Y2I2NmI3ZTg1NGI3NjcyNmQ0ODE4ZTE3YTNhNmM=","created_at":1728915837583}],"acl_mnesia":[{"type":"clientid","type_value":"authz_cid1","topic":"t/1","action":"pub","access":"allow","created_at":1728915852861},{"type":"clientid","type_value":"authz_cid1","topic":"t/1","action":"sub","access":"allow","created_at":1728915852861}],"modules":[{"id":"module:internal_acl","type":"internal_acl","config":{"acl_rule_file":{"filename":"acl.conf","file":"%%--------------------------------------------------------------------\n%% [ACL](https://docs.emqx.io/broker/v3/en/config.html)\n%%\n%% -type(who() :: all | binary() |\n%%                {ipaddr, esockd_access:cidr()} |\n%%                {ipaddrs, [esockd_access:cidr()]} |\n%%                {client, binary()} |\n%%                {user, binary()}).\n%%\n%% -type(access() :: subscribe | publish | pubsub).\n%%\n%% -type(topic() :: binary()).\n%%\n%% -type(rule() :: {allow, all} |\n%%                 {allow, who(), access(), list(topic())} |\n%%                 {deny, all} |\n%%                 {deny, who(), access(), list(topic())}).\n%%--------------------------------------------------------------------\n\n{allow, {user, \"dashboard\"}, subscribe, [\"$SYS/#\"]}.\n\n{allow, {ipaddr, \"127.0.0.1\"}, pubsub, [\"$SYS/#\", \"#\"]}.\n\n{deny, all, subscribe, [\"$SYS/#\", {eq, \"#\"}, {eq, \"+/#\"}]}.\n\n{allow, all}.\n\n"}},"enabled":true,"created_at":{"updated_at_ns":1728915241386230200,"created_at_ns":1728915241386230200},"description":""},{"id":"module:presence","type":"presence","config":{"qos":0},"enabled":true,"created_at":{"updated_at_ns":1728915241386260349,"created_at_ns":1728915241386260349},"description":""},{"id":"module:recon","type":"recon","config":{},"enabled":true,"created_at":{"updated_at_ns":1728915241386298494,"created_at_ns":1728915241386298494},"description":""},{"id":"module:mnesia_authentication","type":"mnesia_authentication","config":{"password_hash":"sha256","max_acls_for_each_login":0},"enabled":true,"created_at":{"updated_at_ns":1728915819392200802,"created_at_ns":1728915819392200802},"description":""},{"id":"module:redis_authentication","type":"redis_authentication","config":{"verify":false,"type":"single","super_cmd":"HGET mqtt_user:%u is_superuser","ssl":false,"server":"redis:6379","sentinel":"","query_timeout":"5s","pool_size":8,"password_hash":"plain","password":"","keyfile":{"filename":"","file":""},"database":0,"certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auto_reconnect":true,"auth_cmd":"HMGET mqtt_user:%u password","acl_cmd":"HGETALL mqtt_acl:%u"},"enabled":true,"created_at":{"updated_at_ns":1728916225117110350,"created_at_ns":1728916225117110350},"description":""},{"id":"module:pgsql_authentication","type":"pgsql_authentication","config":{"verify":false,"user":"postgres","super_query":"select is_superuser from mqtt_user where username = '%u' limit 1","ssl":false,"server":"postgres:5432","query_timeout":"5s","pool_size":8,"password_hash":"sha256","password":"postgres","keyfile":{"filename":"","file":""},"database":"postgres","certfile":{"filename":"","file":""},"cacertfile":{"filename":"","file":""},"auto_reconnect":true,"auth_query":"select password from mqtt_user where username = '%u' limit 1","acl_query":"select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"},"enabled":true,"created_at":{"updated_at_ns":1728916353614061576,"created_at_ns":1728916353614061576},"description":""},{"id":"module:retainer","type":"retainer","config":{"storage_type":"ram","stop_publish_clear_msg":false,"max_retained_messages":0,"max_payload_size":"1MB","expiry_interval":0},"enabled":true,"created_at":{"updated_at_ns":1728916693575495289,"created_at_ns":1728915241386322176},"description":{"zh":"设置 MQTT retain 消息的启用状态、存储位置以及有效期等参数。","en":"Set parameters such as enable status, storage location, and expiration date for MQTT retain messages."}}],"schemas":[],"configs":[],"listeners_state":[],"date":"2024-10-14 16:38:00"}

test/data/auth-builtin-redis-postgres1.md

@@ -0,0 +1,5 @@
+- dashbord user `dash_user`, password `dash_pass`
+- app user `app_id`, name `app_name`, secret `4mVZvVT9CnC6Z3AYdk9C07Ecz9AuBCLblb43kk69BcxbBhP`
+- 3 blacklisted entries: clientid, username, peerhost
+- mnesia, redis, postgres almost-default configs for authn and authz
+- ram retainer default config