Merge pull request #618 from ferd/otp-20-unicode-support

OTP-20 unicode support and OTP-21 readiness

priv/templates/extended_bin

@@ -151,7 +151,7 @@ relx_get_nodename() {
     id="longname$(relx_gen_id)-${NAME}"
     "$BINDIR/erl" -boot start_clean \
         -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-        -eval '[H]=tl(string:tokens(atom_to_list(node()),"@")), io:format("~s~n",[H]), halt()' \
+        -eval '[_,H]=re:split(atom_to_list(node()),"@",[unicode,{return,list}]), io:format("~s~n",[H]), halt()' \
         -noshell ${NAME_TYPE} $id
 }
 

priv/templates/nodetool

@@ -57,7 +57,7 @@ main(Args) ->
             % spaces, so this converts all of that to a single string to parse
             String = binary_to_list(
                       list_to_binary(
-                        string:join(ListOfArgs," ")
+                        join(ListOfArgs," ")
                       )
                     ),
 
@@ -126,16 +126,16 @@ epmd_path() ->
 
 
 nodename(Name) ->
-    case string:tokens(Name, "@") of
+    case re:split(Name, "@", [{return, list}, unicode]) of
         [_Node, _Host] ->
             list_to_atom(Name);
         [Node] ->
-            [_, Host] = string:tokens(atom_to_list(node()), "@"),
+            [_, Host] = re:split(atom_to_list(node()), "@", [{return, list}, unicode]),
             list_to_atom(lists:concat([Node, "@", Host]))
     end.
 
 append_node_suffix(Name, Suffix) ->
-    case string:tokens(Name, "@") of
+    case re:split(Name, "@", [{return, list}, unicode]) of
         [Node, Host] ->
             list_to_atom(lists:concat([Node, Suffix, os:getpid(), "@", Host]));
         [Node] ->
@@ -165,3 +165,13 @@ consult(Cont, Str, Acc) ->
         {more, Cont1} ->
             consult(Cont1, eof, Acc)
     end.
+
+%% string:join/2 copy; string:join/2 is getting obsoleted
+%% and replaced by lists:join/2, but lists:join/2 is too new
+%% for version support (only appeared in 19.0) so it cannot be
+%% used. Instead we just adopt join/2 locally and hope it works
+%% for most unicode use cases anyway.
+join([], Sep) when is_list(Sep) ->
+    [];
+join([H|T], Sep) ->
+    H ++ lists:append([Sep ++ X || X <- T]).

rebar.config

@@ -1,8 +1,8 @@
 %% -*- mode: Erlang; fill-column: 80; comment-column: 75; -*-
 %% Dependencies ================================================================
-{deps, [{erlware_commons, "1.0.0"},
+{deps, [{erlware_commons, "1.0.1"},
         {providers, "1.6.0"},
-        {getopt, "0.8.2"},
+        {getopt, "1.0.1"},
         {cf, "0.2.2"},
         {bbmustache, "1.0.4"}
        ]}.
@@ -17,6 +17,7 @@
     [{platform_define, "^[0-9]+", namespaced_types},
      {platform_define, "^1[8|9]", rand_module},
      {platform_define, "^2", rand_module},
+     {platform_define, "^2", unicode_str},
      warnings_as_errors,
      inline]}.
 
@@ -53,6 +54,7 @@
                                          {erl_opts, [{platform_define, "^[0-9]+", namespaced_types},
                                                      {platform_define, "^R1[4|5]", deprecated_crypto},
                                                      {platform_define, "^((1[8|9])|2)", rand_module},
+                                                     {platform_define, "^2", unicode_str},
                                                      no_debug_info,
                                                      warnings_as_errors
                                                     ]},
@@ -61,7 +63,8 @@
             {override, bbmustache, [
                                    {erl_opts, [no_debug_info]},
                                    {deps, []}, {plugins, []}]},
-            {override, getopt, [{erl_opts, [no_debug_info]}]},
+            {override, getopt, [{erl_opts, [no_debug_info,
+                                            {platform_define, "^2", unicode_str}]}]},
             {override, providers, [{erl_opts, [no_debug_info]}]}
             ]}.
 

rebar.lock

@@ -1,14 +1,14 @@
 {"1.1.0",
 [{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.0.4">>},0},
  {<<"cf">>,{pkg,<<"cf">>,<<"0.2.2">>},0},
- {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.0.0">>},0},
- {<<"getopt">>,{pkg,<<"getopt">>,<<"0.8.2">>},0},
+ {<<"erlware_commons">>,{pkg,<<"erlware_commons">>,<<"1.0.1">>},0},
+ {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0},
  {<<"providers">>,{pkg,<<"providers">>,<<"1.6.0">>},0}]}.
 [
 {pkg_hash,[
  {<<"bbmustache">>, <<"7BA94F971C5AFD7B6617918A4BB74705E36CAB36EB84B19B6A1B7EE06427AA38">>},
  {<<"cf">>, <<"7F2913FFF90ABCABD0F489896CFEB0B0674F6C8DF6C10B17A83175448029896C">>},
- {<<"erlware_commons">>, <<"087467DE5833C0BB5B3CCDD387F9E9C1FB816A75B7A709629BF24B5ED3246C51">>},
- {<<"getopt">>, <<"B17556DB683000BA50370B16C0619DF1337E7AF7ECBF7D64FBF8D1D6BCE3109B">>},
+ {<<"erlware_commons">>, <<"ABC13522C826CA709173FA20DBBF7C7D00ADE914A770A1364962A1DF8FE98DE5">>},
+ {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>},
  {<<"providers">>, <<"DB0E2F9043AE60C0155205FCD238D68516331D0E5146155E33D1E79DC452964A">>}]}
 ].

src/rlx_app_discovery.erl

@@ -40,7 +40,7 @@ do(State, LibDirs) ->
     ec_cmd_log:info(rlx_state:log(State),
                     fun() ->
                             ["Resolving OTP Applications from directories:\n",
-                             string:join([[rlx_util:indent(2), LibDir] || LibDir <- LibDirs], "\n")]
+                             rlx_string:join([[rlx_util:indent(2), LibDir] || LibDir <- LibDirs], "\n")]
                     end),
     resolve_app_metadata(State, LibDirs).
 

src/rlx_prv_app_discover.erl

@@ -132,7 +132,7 @@ add_system_lib_dir(State) ->
 add_environment_lib_dir(_State) ->
     case os:getenv("ERL_LIBS") of
         false -> [];
-        Libs -> [erlang:iolist_to_binary(L) || L <- string:tokens(Libs, ":")]
+        Libs -> [erlang:iolist_to_binary(L) || L <- rlx_string:lexemes(Libs, ":")]
     end.
 
 %% Order matters so this slow dedup needs to be used

src/rlx_prv_assembler.erl

@@ -111,12 +111,12 @@ format_error({unable_to_make_symlink, AppDir, TargetDir, Reason}) ->
     io_lib:format("Unable to symlink directory ~s to ~s because \n~s~s",
                   [AppDir, TargetDir, rlx_util:indent(2),
                    file:format_error(Reason)]);
-format_error(start_clean_script_generation_error) ->
+format_error(boot_script_generation_error) ->
     "Unknown internal release error generating start_clean.boot";
-format_error({start_clean_script_generation_warning, Module, Warnings}) ->
+format_error({boot_script_generation_warning, Module, Warnings}) ->
     ["Warnings generating start_clean.boot \s",
      rlx_util:indent(2), Module:format_warning(Warnings)];
-format_error({start_clean_script_generation_error, Module, Errors}) ->
+format_error({boot_script_generation_error, Module, Errors}) ->
     ["Errors generating start_clean.boot \n",
      rlx_util:indent(2), Module:format_error(Errors)];
 format_error({strip_release, Reason}) ->
@@ -354,17 +354,22 @@ create_release_info(State0, Release0, OutputDir) ->
     ReleaseDir = rlx_util:release_output_dir(State0, Release0),
     ReleaseFile = filename:join([ReleaseDir, RelName ++ ".rel"]),
     StartCleanFile = filename:join([ReleaseDir, "start_clean.rel"]),
+    NoDotErlFile = filename:join([ReleaseDir, "no_dot_erlang.rel"]),
     ok = ec_file:mkdir_p(ReleaseDir),
     Release1 = rlx_release:relfile(Release0, ReleaseFile),
     State1 = rlx_state:update_realized_release(State0, Release1),
     case rlx_release:metadata(Release1) of
         {ok, Meta} ->
-            case rlx_release:start_clean_metadata(Release1) of
-                {ok, StartCleanMeta} ->
+            case {rlx_release:start_clean_metadata(Release1),
+                  rlx_release:no_dot_erlang_metadata(Release1)} of
+                {{ok, StartCleanMeta}, {ok, NoDotErlMeta}} ->
                     ok = ec_file:write_term(ReleaseFile, Meta),
                     ok = ec_file:write_term(StartCleanFile, StartCleanMeta),
+                    ok = ec_file:write_term(NoDotErlFile, NoDotErlMeta),
                     write_bin_file(State1, Release1, OutputDir, ReleaseDir);
-                E ->
+                {{ok, _}, E} ->
+                    E;
+                {_, E} ->
                     E
             end;
         E ->
@@ -416,13 +421,13 @@ write_bin_file(State, Release, OutputDir, RelDir) ->
         _ ->
             VsnRelStartFile = case OsFamily of
                 unix -> VsnRel;
-                win32 -> string:concat(VsnRel, ".cmd")
+                win32 -> rlx_string:concat(VsnRel, ".cmd")
             end,
             ok = file:write_file(VsnRelStartFile, StartFile),
             ok = file:change_mode(VsnRelStartFile, 8#777),
             BareRelStartFile = case OsFamily of
                 unix -> BareRel;
-                win32 -> string:concat(BareRel, ".cmd")
+                win32 -> rlx_string:concat(BareRel, ".cmd")
             end,
             ok = file:write_file(BareRelStartFile, StartFile),
             ok = file:change_mode(BareRelStartFile, 8#777)
@@ -494,15 +499,15 @@ hook_filename(builtin_status) -> "hooks/builtin/status".
 hook_invocation({custom, CustomScript}) -> CustomScript;
 %% the pid builtin hook with no arguments writes to pid file
 %% at /var/run/{{ rel_name }}.pid
-hook_invocation(pid) -> string:join(["hooks/builtin/pid",
+hook_invocation(pid) -> rlx_string:join(["hooks/builtin/pid",
                                      "/var/run/$REL_NAME.pid"], "|");
-hook_invocation({pid, PidFile}) -> string:join(["hooks/builtin/pid",
+hook_invocation({pid, PidFile}) -> rlx_string:join(["hooks/builtin/pid",
                                                 PidFile], "|");
 hook_invocation(wait_for_vm_start) -> "hooks/builtin/wait_for_vm_start";
 hook_invocation({wait_for_process, Name}) ->
     %% wait_for_process takes an atom as argument
     %% which is the process name to wait for
-    string:join(["hooks/builtin/wait_for_process",
+    rlx_string:join(["hooks/builtin/wait_for_process",
                  atom_to_list(Name)], "|");
 hook_invocation(builtin_status) -> "hooks/builtin/status".
 
@@ -692,13 +697,15 @@ make_boot_script(State, Release, OutputDir, RelDir) ->
             ec_cmd_log:info(rlx_state:log(State),
                              "release successfully created!"),
             create_RELEASES(OutputDir, ReleaseFile),
+            create_no_dot_erlang(RelDir, OutputDir, Options, State),
             create_start_clean(RelDir, OutputDir, Options, State);
         error ->
             ?RLX_ERROR({release_script_generation_error, ReleaseFile});
         {ok, _, []} ->
             ec_cmd_log:info(rlx_state:log(State),
                           "release successfully created!"),
             create_RELEASES(OutputDir, ReleaseFile),
+            create_no_dot_erlang(RelDir, OutputDir, Options, State),
             create_start_clean(RelDir, OutputDir, Options, State);
         {ok,Module,Warnings} ->
             ?RLX_ERROR({release_script_generation_warn, Module, Warnings});
@@ -728,29 +735,35 @@ make_boot_script_variables(State) ->
             [{"ERTS_LIB_DIR", code:lib_dir()}]
     end.
 
+create_no_dot_erlang(RelDir, OutputDir, Options, State) ->
+    create_boot_file(RelDir, OutputDir, Options, State, "no_dot_erlang").
+
 create_start_clean(RelDir, OutputDir, Options, State) ->
+    create_boot_file(RelDir, OutputDir, Options, State, "start_clean").
+
+create_boot_file(RelDir, OutputDir, Options, State, Name) ->
     case rlx_util:make_script(Options,
                          fun(CorrectedOptions) ->
-                                 systools:make_script("start_clean", CorrectedOptions)
+                                 systools:make_script(Name, CorrectedOptions)
                          end) of
         ok ->
-            ok = ec_file:copy(filename:join([RelDir, "start_clean.boot"]),
-                              filename:join([OutputDir, "bin", "start_clean.boot"])),
-            ec_file:remove(filename:join([RelDir, "start_clean.rel"])),
-            ec_file:remove(filename:join([RelDir, "start_clean.script"])),
+            ok = ec_file:copy(filename:join([RelDir, Name++".boot"]),
+                              filename:join([OutputDir, "bin", Name++".boot"])),
+            ec_file:remove(filename:join([RelDir, Name++".rel"])),
+            ec_file:remove(filename:join([RelDir, Name++".script"])),
             {ok, State};
         error ->
-            ?RLX_ERROR(start_clean_script_generation_error);
+            ?RLX_ERROR(boot_script_generation_error);
         {ok, _, []} ->
-            ok = ec_file:copy(filename:join([RelDir, "start_clean.boot"]),
-                              filename:join([OutputDir, "bin", "start_clean.boot"])),
-            ec_file:remove(filename:join([RelDir, "start_clean.rel"])),
-            ec_file:remove(filename:join([RelDir, "start_clean.script"])),
+            ok = ec_file:copy(filename:join([RelDir, Name++".boot"]),
+                              filename:join([OutputDir, "bin", Name++".boot"])),
+            ec_file:remove(filename:join([RelDir, Name++".rel"])),
+            ec_file:remove(filename:join([RelDir, Name++".script"])),
             {ok, State};
         {ok,Module,Warnings} ->
-            ?RLX_ERROR({start_clean_script_generation_warn, Module, Warnings});
+            ?RLX_ERROR({boot_script_generation_warn, Module, Warnings});
         {error,Module,Error} ->
-            ?RLX_ERROR({start_clean_script_generation_error, Module, Error})
+            ?RLX_ERROR({boot_script_generation_error, Module, Error})
     end.
 
 create_RELEASES(OutputDir, ReleaseFile) ->
@@ -796,15 +809,15 @@ extended_bin_file_contents(OsFamily, RelName, RelVsn, ErtsVsn, ErlOpts, Hooks, E
         win32 -> extended_bin_windows
     end,
     %% turn all the hook lists into space separated strings
-    PreStartHooks = string:join(proplists:get_value(pre_start, Hooks, []), " "),
-    PostStartHooks = string:join(proplists:get_value(post_start, Hooks, []), " "),
-    PreStopHooks = string:join(proplists:get_value(pre_stop, Hooks, []), " "),
-    PostStopHooks = string:join(proplists:get_value(post_stop, Hooks, []), " "),
-    PreInstallUpgradeHooks = string:join(proplists:get_value(pre_install_upgrade,
+    PreStartHooks = rlx_string:join(proplists:get_value(pre_start, Hooks, []), " "),
+    PostStartHooks = rlx_string:join(proplists:get_value(post_start, Hooks, []), " "),
+    PreStopHooks = rlx_string:join(proplists:get_value(pre_stop, Hooks, []), " "),
+    PostStopHooks = rlx_string:join(proplists:get_value(post_stop, Hooks, []), " "),
+    PreInstallUpgradeHooks = rlx_string:join(proplists:get_value(pre_install_upgrade,
                                                 Hooks, []), " "),
-    PostInstallUpgradeHooks = string:join(proplists:get_value(post_install_upgrade,
+    PostInstallUpgradeHooks = rlx_string:join(proplists:get_value(post_install_upgrade,
                                                  Hooks, []), " "),
-    StatusHook = string:join(proplists:get_value(status, Hooks, []), " "),
+    StatusHook = rlx_string:join(proplists:get_value(status, Hooks, []), " "),
     {ExtensionsList1, ExtensionDeclarations1} =
         lists:foldl(fun({Name, Script},
                         {ExtensionsList0, ExtensionDeclarations0}) ->
@@ -816,10 +829,10 @@ extended_bin_file_contents(OsFamily, RelName, RelVsn, ErtsVsn, ErlOpts, Hooks, E
                     end, {[], []}, Extensions),
     % pipe separated string of extensions, to show on the start script usage
     % (eg. foo|bar)
-    ExtensionsList = string:join(ExtensionsList1 ++ ["undefined"], "|"),
+    ExtensionsList = rlx_string:join(ExtensionsList1 ++ ["undefined"], "|"),
     % command separated string of extension script declarations
     % (eg. foo_extension="path/to/foo_script")
-    ExtensionDeclarations = string:join(ExtensionDeclarations1, ";"),
+    ExtensionDeclarations = rlx_string:join(ExtensionDeclarations1, ";"),
     render(Template, [{rel_name, RelName}, {rel_vsn, RelVsn},
                       {erts_vsn, ErtsVsn}, {erl_opts, ErlOpts},
                       {pre_start_hooks, PreStartHooks},
@@ -833,7 +846,7 @@ extended_bin_file_contents(OsFamily, RelName, RelVsn, ErtsVsn, ErlOpts, Hooks, E
                       {extension_declarations, ExtensionDeclarations}]).
 
 erl_ini(OutputDir, ErtsVsn) ->
-    ErtsDirName = string:concat("erts-", ErtsVsn),
+    ErtsDirName = rlx_string:concat("erts-", ErtsVsn),
     BinDir = filename:join([OutputDir, ErtsDirName, bin]),
     render(erl_ini, [{bin_dir, BinDir}, {output_dir, OutputDir}]).
 

src/rlx_prv_release.erl

@@ -202,7 +202,7 @@ set_resolved(State, Release0, Pkgs) ->
                 ErtsDir ->
                     try
                         [Erts | _] = filelib:wildcard(filename:join(ErtsDir, "erts-*")),
-                        [_, ErtsVsn] = string:tokens(filename:basename(Erts), "-"),
+                        [_, ErtsVsn] = rlx_string:lexemes(filename:basename(Erts), "-"),
                         {ok, rlx_state:add_realized_release(State, rlx_release:erts(Release1, ErtsVsn))}
                     catch
                         _:_ ->

src/rlx_rel_discovery.erl

@@ -42,7 +42,7 @@ do(State, LibDirs, AppMeta) ->
             ec_cmd_log:info(rlx_state:log(State),
                             fun() ->
                                     ["Resolving available OTP Releases from directories:\n",
-                                     string:join([[rlx_util:indent(2), LibDir] || LibDir <- LibDirs], "\n")]
+                                     rlx_string:join([[rlx_util:indent(2), LibDir] || LibDir <- LibDirs], "\n")]
                             end),
             resolve_rel_metadata(State, LibDirs, AppMeta)
     end.

src/rlx_release.erl

@@ -39,6 +39,7 @@
          realized/1,
          metadata/1,
          start_clean_metadata/1,
+         no_dot_erlang_metadata/1,
          canonical_name/1,
          config/1,
          config/2,
@@ -198,6 +199,12 @@ start_clean_metadata(#release_t{name=Name, vsn=Vsn, erts=ErtsVsn, applications=A
             ?RLX_ERROR({not_realized, Name, Vsn})
     end.
 
+%% The no_dot_erlang.rel.src file is a literal copy of start_clean.rel.src
+%% in Erlang/OTP itself.
+-spec no_dot_erlang_metadata(t()) -> term().
+no_dot_erlang_metadata(T) ->
+    start_clean_metadata(T).
+
 %% @doc produce the canonical name (<name>-<vsn>) for this release
 -spec canonical_name(t()) -> string().
 canonical_name(#release_t{name=Name, vsn=Vsn}) ->

src/rlx_string.erl

@@ -0,0 +1,23 @@
+%% Compatibility module for the string API changes between
+%% OTP-19 and OTP-21, where Unicode support means the deprecation
+%% of a lot of string functions.
+-module(rlx_string).
+-export([concat/2, lexemes/2, join/2]).
+
+-ifdef(unicode_str).
+concat(Str1, Str2) -> unicode:characters_to_list([Str1,Str2]).
+lexemes(Str, Separators) -> string:lexemes(Str, Separators).
+-else.
+concat(Str1, Str2) -> string:concat(Str1, Str2).
+lexemes(Str, Separators) -> string:tokens(Str, Separators).
+-endif.
+
+%% string:join/2 copy; string:join/2 is getting obsoleted
+%% and replaced by lists:join/2, but lists:join/2 is too new
+%% for version support (only appeared in 19.0) so it cannot be
+%% used. Instead we just adopt join/2 locally and hope it works
+%% for most unicode use cases anyway.
+join([], Sep) when is_list(Sep) ->
+    [];
+join([H|T], Sep) ->
+    H ++ lists:append([Sep ++ X || X <- T]).