Fix acl file import

README.md

@@ -76,8 +76,6 @@ After convertion, the output tarball contains a structure similar to this:
 
 ```
 /tmp/emqx-export-2024-10-10-14-27-58.171/
-├── authz
-│   └── acl.conf
 ├── cluster.hocon
 ├── META.hocon
 └── mnesia
@@ -89,24 +87,14 @@ After convertion, the output tarball contains a structure similar to this:
     └── emqx_retainer_message
 ```
 
-- `authz/acl.conf` must be copied to `$EMQX_DATA_DIR/authz/acl.conf`, where `$EMQX_DATA_DIR` is EMQX's data directory and depends on the installation method and system (typically `/var/lib/emqx/data` or `/opt/emqx/data`).
-- `cluster.hocon` must be copied to `$EMQX_DATA_DIR/configs/cluster.hocon`.
-- Each file in `mnesia` dir must be imported with the following command:
+> [!TIP]
+> There's no need to extract this file.  Just place it somewhere where `emqx` application
+> user may read it.
 
-  ```sh
-  emqx eval 'mnesia:restore("/full/path/to/file", []).'
-  ```
-
-  So, for example:
+- To import this file, simply run:
 
   ```sh
-  emqx eval 'mnesia:restore("/tmp/emqx-export-2024-10-10-14-27-58.171/mnesia/emqx_authn_mnesia", []).'
-  ```
-
-  Yields, when successful:
-
-  ```
-  {atomic, [emqx_authn_mnesia]}
+  emqx ctl data import full/path/to/emqx-export-2024-10-10-14-27-58.171.tar.gz
   ```
 
 - If you have retained messages being imported (from `mnesia/emqx_retainer_message`), then, after importing that table with the command above, you must run:

src/emqx_data_converter.erl

@@ -150,7 +150,7 @@ main1(Opts) ->
     OutRawConfRule = convert_rules_resources(InputMap, OutRawConf),
     {BackupName, TarDescriptor} = prepare_new_backup(OutputDir),
     Edition = proplists:get_value(edition, Opts),
-    {ok, BackupTarName} = export(OutRawConfRule, BackupName, TarDescriptor, Edition, OutputDir),
+    {ok, BackupTarName} = export(OutRawConfRule, BackupName, TarDescriptor, Edition),
     file:del_dir_r(BackupName),
     log_info("Converted to EMQX 5.1 backup file: ~s", [BackupTarName]).
 
@@ -212,7 +212,7 @@ prepare_new_backup(OutputDir) ->
     {ok, TarDescriptor} = ?fmt_tar_err(erl_tar:open(BackupTarName, [write, compressed])),
     {BackupName, TarDescriptor}.
 
-export(OutRawConf, BackupName, TarDescriptor, Edition, OutputDir) ->
+export(OutRawConf, BackupName, TarDescriptor, Edition) ->
     BackupBaseName = filename:basename(BackupName),
     BackupTarName = ?tar(BackupName),
     Meta = #{
@@ -225,23 +225,10 @@ export(OutRawConf, BackupName, TarDescriptor, Edition, OutputDir) ->
     ok = export_mnesia_tabs(TarDescriptor, BackupName, BackupBaseName),
     RawConfBin = bin(hocon_pp:do(OutRawConf, #{})),
     ConfNameInArchive = filename:join(BackupBaseName, ?CLUSTER_HOCON_FILENAME),
-    ok = add_acl_conf_file(TarDescriptor, BackupBaseName, OutputDir),
     ok = ?fmt_tar_err(erl_tar:add(TarDescriptor, RawConfBin, ConfNameInArchive, [])),
     ok = ?fmt_tar_err(erl_tar:close(TarDescriptor)),
     {ok, BackupTarName}.
 
-add_acl_conf_file(TarDescriptor, BackupBaseName, OutputDir) ->
-    TmpFile = output_acl_file_path(OutputDir),
-    case filelib:is_regular(TmpFile) of
-        true ->
-            NameInArchive = filename:join([BackupBaseName, "authz", "acl.conf"]),
-            ok = ?fmt_tar_err(erl_tar:add(TarDescriptor, TmpFile, NameInArchive, [])),
-            file:delete(TmpFile),
-            ok;
-        false ->
-            ok
-    end.
-
 export_mnesia_tabs(TarDescriptor, BackupName, BackupBaseName) ->
     lists:foreach(
         fun(Tab) -> export_mnesia_tab(TarDescriptor, Tab, BackupName, BackupBaseName) end,
@@ -1208,10 +1195,8 @@ convert_acl_rules(_IsEnabled, <<>> = _AclRules, _Opts) ->
     log_warning("Skipping ACL file authorization, as ACL file content is empty."),
     undefined;
 convert_acl_rules(IsEnabled, AclRulesBin, Opts) ->
-    OutputDir = maps:get(output_dir, Opts),
-    TmpFile = output_acl_file_path(OutputDir),
+    TmpFile = filename:join(maps:get(output_dir, Opts), "acl.conf"),
     try
-        ok = filelib:ensure_dir(TmpFile),
         ok = file:write_file(TmpFile, AclRulesBin),
         {ok, AclRules0} = file:consult(TmpFile),
         AclRules = lists:map(
@@ -1222,21 +1207,20 @@ convert_acl_rules(IsEnabled, AclRulesBin, Opts) ->
                              io_lib:format("~p.~n~n", [AllRule])
                      end,
                      AclRules0),
-        ok = file:write_file(TmpFile, [?ACL_FILE_COMMENTS | AclRules]),
+        AclRules1 = iolist_to_binary([?ACL_FILE_COMMENTS | AclRules]),
         #{<<"enable">> => IsEnabled,
           <<"type">> => <<"file">>,
-          <<"path">> => filename:join(["data", "authz", "acl.conf"])}
+          <<"rules">> => AclRules1}
     catch
         _:Reason:St ->
             log_error(
-              "failed to convert ACL file, reason: ~p, stacktrace:\n  ~p",
+              "failed to convert ACL file, reason: ~p, stacktrace: ~p",
               [Reason, St]),
             undefined
+    after
+        file:delete(TmpFile)
     end.
 
-output_acl_file_path(OutputDir) ->
-    filename:join([OutputDir, "tmp", "authz", "acl.conf"]).
-
 convert_access_type(subscribe) -> subscribe;
 convert_access_type(publish) -> publish;
 convert_access_type(pubsub) -> all.

test/scripts/test-convert-and-load.exs

@@ -49,32 +49,13 @@
     end
   end
 
-  def place_files(input_filepath, outdir) do
-    Logger.info(%{input: input_filepath, msg: "importing_input"})
-    File.mkdir_p!("converted")
-    {:ok, _} = run("bash", ["-c", "tar -C converted -xvf #{input_filepath} --strip-components=1"])
-
-    {:ok, _} =
-      run_in_container([
-        "cp #{outdir}/cluster.hocon data/configs/cluster.hocon",
-        "mkdir -p data/authz/",
-        "cp #{outdir}/authz/acl.conf data/authz/acl.conf"
-      ])
-
-    :ok
-  end
-
   def import!(backup_filepath) do
     basename = Path.basename(backup_filepath)
     container_filepath = "/tmp/#{basename}"
     {:ok, _} = docker_cp(backup_filepath, container_filepath)
     Logger.info(%{msg: "starting_emqx"})
     :ok = start_emqx()
 
-    # FIXME: after patching `emqx ctl data import` in EMQX itself to support placing
-    # `acl.conf` in the reight automatically, this won't be necessary.
-    place_acl_conf_file(container_filepath)
-
     {:ok, output} =
       run_in_container("docker-entrypoint.sh emqx ctl data import #{container_filepath}")
 
@@ -87,23 +68,6 @@
     end
   end
 
-  # temporary hack (5.8.1 doesn't handle this new file)
-  def place_acl_conf_file(container_filepath) do
-    extracted_dir = "/tmp/extracted"
-
-    {:ok, output} =
-      run_in_container([
-        "mkdir -p #{extracted_dir}",
-        "tar -C #{extracted_dir} -xvf #{container_filepath}",
-        "mkdir -p data/authz/",
-        "cp #{extracted_dir}/*/authz/acl.conf data/authz/acl.conf"
-      ])
-
-    IO.puts(output)
-
-    :ok
-  end
-
   def start_emqx(opts \\ []) do
     wait_s = Keyword.get(opts, :wait_s, 15)
 
@@ -284,7 +248,7 @@
     _Expired = undefined).
   ```
   """
   def api_req!(method, path, body \\ "", opts \\ []) do
     api_user = "app_id"
     api_pass = "4mVZvVT9CnC6Z3AYdk9C07Ecz9AuBCLblb43kk69BcxbBhP"
     authn64 = Base.encode64("#{api_user}:#{api_pass}")
@@ -326,7 +290,6 @@
       outdir = "/opt/data/converted"
       path = "test/data/barebones.json"
       {:ok, converted_path} = TH.convert!(path)
-      TH.place_files(converted_path, outdir)
 
   After this, `docker exec -it emqx-data-converter bash`