From 054f39e7b7ba9530e5fd22548fbd6eddae88bc19 Mon Sep 17 00:00:00 2001 From: Evan Vigil-McClanahan Date: Fri, 24 Sep 2021 10:15:07 -0700 Subject: [PATCH] add support for optional_applications in otp24 Optional applications may or may not also be in the applications list. They are also only optional for the app that lists them as optional, unlike an excluded application which is global. So what is optional for app A may be required by app B and thus cause a failure if it is not found when building the release. --- rebar.config | 1 + src/rlx_app_info.erl | 21 ++++++- src/rlx_assemble.erl | 19 ++++-- src/rlx_resolve.erl | 66 ++++++++++++-------- test/rlx_release_SUITE.erl | 125 ++++++++++++++++++++++++++++++++++++- test/rlx_test_utils.erl | 35 +++++++---- 6 files changed, 220 insertions(+), 47 deletions(-) diff --git a/rebar.config b/rebar.config index e7ce0255f..60b261e66 100644 --- a/rebar.config +++ b/rebar.config @@ -45,6 +45,7 @@ %% used in tests {rlx_app_info, new, 5}, {rlx_app_info, new, 6}, + {rlx_app_info, new, 7}, {rlx_file_utils, type, 1}, {rlx_file_utils, write, 2}, {rlx_release, applications, 1}, diff --git a/src/rlx_app_info.erl b/src/rlx_app_info.erl index 6f2774c6c..0ac20c1d4 100644 --- a/src/rlx_app_info.erl +++ b/src/rlx_app_info.erl @@ -38,11 +38,13 @@ -export([new/5, new/6, + new/7, name/1, vsn/1, dir/1, applications/1, included_applications/1, + optional_applications/1, link/1, link/2, format_error/1, @@ -57,6 +59,7 @@ applications := [atom()], included_applications := [atom()], + optional_applications := [atom()], dir := file:name() | undefined, link := boolean() | undefined, @@ -73,15 +76,23 @@ -spec new(atom(), string(), file:name(), [atom()], [atom()]) -> t(). new(Name, Vsn, Dir, Applications, IncludedApplications) -> - new(Name, Vsn, Dir, Applications, IncludedApplications, dep). + new(Name, Vsn, Dir, Applications, IncludedApplications, [], dep). --spec new(atom(), string(), file:name(), [atom()], [atom()], app_type()) -> t(). -new(Name, Vsn, Dir, Applications, IncludedApplications, AppType) -> +-spec new(atom(), string(), file:name(), [atom()], [atom()], [atom()] | atom()) -> t(). +new(Name, Vsn, Dir, Applications, IncludedApplications, OptionalApplications) + when is_list(OptionalApplications) -> + new(Name, Vsn, Dir, Applications, IncludedApplications, OptionalApplications, dep); +new(Name, Vsn, Dir, Applications, IncludedApplications, AppType) when is_atom(AppType) -> + new(Name, Vsn, Dir, Applications, IncludedApplications, [], AppType). + +-spec new(atom(), string(), file:name(), [atom()], [atom()], [atom()], app_type()) -> t(). +new(Name, Vsn, Dir, Applications, IncludedApplications, OptionalApplications, AppType) -> #{name => Name, vsn => Vsn, applications => Applications, included_applications => IncludedApplications, + optional_applications => OptionalApplications, dir => Dir, link => false, @@ -108,6 +119,10 @@ applications(#{applications := Deps}) -> included_applications(#{included_applications := Deps}) -> Deps. +-spec optional_applications(t()) -> [atom()]. +optional_applications(#{included_applications := Deps}) -> + Deps. + -spec link(t()) -> boolean(). link(#{link := Link}) -> Link. diff --git a/src/rlx_assemble.erl b/src/rlx_assemble.erl index ad21a4aa8..5a0230b0c 100644 --- a/src/rlx_assemble.erl +++ b/src/rlx_assemble.erl @@ -129,6 +129,7 @@ rewrite_app_file(State, App, TargetDir) -> Name = rlx_app_info:name(App), Applications = rlx_app_info:applications(App), IncludedApplications = rlx_app_info:included_applications(App), + OptionalApplications = rlx_app_info:optional_applications(App), %% TODO: should really read this in when creating rlx_app:t() and keep it AppFile = filename:join([TargetDir, "ebin", [Name, ".app"]]), @@ -141,7 +142,7 @@ rewrite_app_file(State, App, TargetDir) -> end, %% maybe replace excluded apps - AppData1 = maybe_exclude_apps(Applications, IncludedApplications, + AppData1 = maybe_exclude_apps(Applications, IncludedApplications, OptionalApplications, AppData0, rlx_state:exclude_apps(State)), AppData2 = maybe_exclude_modules(AppData1, proplists:get_value(Name, @@ -156,17 +157,23 @@ rewrite_app_file(State, App, TargetDir) -> erlang:error(?RLX_ERROR({rewrite_app_file, AppFile, Error})) end. -maybe_exclude_apps(_Applications, _IncludedApplications, AppData, []) -> +maybe_exclude_apps(_Applications, _IncludedApplications, _OptionalApplications, AppData, []) -> AppData; -maybe_exclude_apps(Applications, IncludedApplications, AppData, ExcludeApps) -> +maybe_exclude_apps(Applications, IncludedApplications, OptionalApplications, AppData, ExcludeApps) -> AppData1 = lists:keyreplace(applications, 1, AppData, {applications, Applications -- ExcludeApps}), - lists:keyreplace(included_applications, + AppData2 = lists:keyreplace(included_applications, + 1, + AppData1, + {included_applications, IncludedApplications -- ExcludeApps}), + + %% not absolutely necessary since they are already seen as optional, but may as well + lists:keyreplace(optional_applications, 1, - AppData1, - {included_applications, IncludedApplications -- ExcludeApps}). + AppData2, + {optional_applications, OptionalApplications -- ExcludeApps}). maybe_exclude_modules(AppData, []) -> AppData; diff --git a/src/rlx_resolve.erl b/src/rlx_resolve.erl index bf23085c0..36f1b472f 100644 --- a/src/rlx_resolve.erl +++ b/src/rlx_resolve.erl @@ -51,39 +51,53 @@ solve_release(Release, State0) -> %% find the app_info records for each application and its deps needed for the release subset(Goals, World, LibDirs, CheckCodeLibDirs, ExcludeApps) -> - {Apps, _} = fold_apps(Goals, World, sets:new(), LibDirs, CheckCodeLibDirs, ExcludeApps), + {Apps, _} = fold_apps(Goals, World, sets:new(), LibDirs, CheckCodeLibDirs, [], ExcludeApps), Apps. -subset(Goal, World, Seen, LibDirs, CheckCodeLibDirs, ExcludeApps) -> +subset(Goal, World, Seen, LibDirs, CheckCodeLibDirs, OptionalApplications, ExcludeApps) -> {Name, Vsn} = name_version(Goal), case sets:is_element(Name, Seen) of true -> {[], Seen}; _ -> - AppInfo=#{applications := Applications, - included_applications := IncludedApplications} = - find_app(Name, Vsn, World, LibDirs, CheckCodeLibDirs), - {Apps, Seen2} = fold_apps(Applications ++ IncludedApplications, - World, - sets:add_element(Name, Seen), - LibDirs, - CheckCodeLibDirs, - ExcludeApps), - - %% don't add excluded apps - %% TODO: should an excluded app's deps also be excluded? - case lists:member(Name, ExcludeApps) of - true -> - {Apps, Seen2}; - false -> - %% place the deps of the App before it - {Apps ++ [AppInfo], Seen2} + case find_app(Name, Vsn, World, LibDirs, CheckCodeLibDirs) of + not_found -> + case lists:member(Name, OptionalApplications) of + true -> + %% don't add to Seen since optional applications are only + %% per-application and not global, so another app could + %% depend on this dependency + {[], Seen}; + false -> + erlang:error(?RLX_ERROR({app_not_found, Name, Vsn})) + end; + AppInfo=#{applications := Applications, + included_applications := IncludedApplications, + optional_applications := OptionalApplications0} -> + {Apps, Seen2} = fold_apps(Applications ++ IncludedApplications ++ OptionalApplications0, + World, + sets:add_element(Name, Seen), + LibDirs, + CheckCodeLibDirs, + OptionalApplications0, + ExcludeApps), + + %% don't add excluded apps + %% TODO: should an excluded app's deps also be excluded? + case lists:member(Name, ExcludeApps) of + true -> + {Apps, Seen2}; + false -> + %% place the deps of the App before it + {Apps ++ [AppInfo], Seen2} + end end end. -fold_apps(Apps, World, Seen, LibDirs, CheckCodeLibDirs, ExcludeApps) -> +fold_apps(Apps, World, Seen, LibDirs, CheckCodeLibDirs, OptionalApplications, ExcludeApps) -> lists:foldl(fun(App, {AppAcc, SeenAcc}) -> - {NewApps, SeenAcc1} = subset(App, World, SeenAcc, LibDirs, CheckCodeLibDirs, ExcludeApps), + {NewApps, SeenAcc1} = subset(App, World, SeenAcc, LibDirs, + CheckCodeLibDirs, OptionalApplications, ExcludeApps), %% put new apps after the existing list to keep the user defined order {AppAcc ++ NewApps, SeenAcc1} end, {[], Seen}, Apps). @@ -146,7 +160,7 @@ search_for_app(Name, Vsn, LibDirs, CheckCodeLibDirs) -> %% app not found in any lib dir we are configured to search %% and user set a custom `system_libs' directory so we do %% not look in `code:lib_dir' - erlang:error(?RLX_ERROR({app_not_found, Name, Vsn})); + not_found; AppInfo -> case check_app(Name, Vsn, AppInfo) of true -> @@ -177,13 +191,13 @@ find_app_in_dir(Name, Vsn, [Dir | Rest]) -> find_app_in_code_path(Name, Vsn) -> case code:lib_dir(Name) of {error, bad_name} -> - erlang:error(?RLX_ERROR({app_not_found, Name, Vsn})); + not_found; Dir -> case to_app(Name, Vsn, filename:join([Dir, "ebin", [Name, ".app"]])) of {true, AppInfo} -> AppInfo; false -> - erlang:error(?RLX_ERROR({app_not_found, Name, Vsn})) + not_found end end. @@ -202,6 +216,7 @@ to_app(Name, Vsn, AppFilePath) -> end, Applications = proplists:get_value(applications, AppData, []), IncludedApplications = proplists:get_value(included_applications, AppData, []), + OptionalApplications = proplists:get_value(optional_applications, AppData, []), case lists:keyfind(vsn, 1, AppData) of {_, Vsn1} when Vsn =:= undefined ; @@ -211,6 +226,7 @@ to_app(Name, Vsn, AppFilePath) -> applications => Applications, included_applications => IncludedApplications, + optional_applications => OptionalApplications, dir => filename:dirname(filename:dirname(AppFilePath)), link => false}}; diff --git a/test/rlx_release_SUITE.erl b/test/rlx_release_SUITE.erl index 783d76a4d..e1eb90ae6 100644 --- a/test/rlx_release_SUITE.erl +++ b/test/rlx_release_SUITE.erl @@ -44,7 +44,9 @@ groups() -> make_exclude_app_release, make_overridden_release, make_goalless_release, make_one_app_top_level_release, make_release_twice, make_erts_release, make_erts_config_release, make_included_nodetool_release, make_release_goal_reorder, - make_release_keep_goal_order]}, + make_release_keep_goal_order, optional_application, missing_optional_application, + missing_duplicated_optional_application, duplicated_optional_applications, + excluded_optional_applications]}, {dev_mode, [shuffle], [make_dev_mode_template_release, make_dev_mode_release, make_release_twice_dev_mode]}, @@ -61,6 +63,14 @@ init_per_suite(Config) -> rlx_test_utils:create_app(LibDir, "goal_app_2", "0.0.1", [stdlib,kernel,goal_app_1,non_goal_2], []), rlx_test_utils:create_app(LibDir, "non_goal_1", "0.0.1", [stdlib,kernel], [lib_dep_1]), rlx_test_utils:create_app(LibDir, "non_goal_2", "0.0.1", [stdlib,kernel], []), + rlx_test_utils:create_app(LibDir, "non_goal_3", "0.0.1", [stdlib,kernel,non_goal_2], []), + rlx_test_utils:create_app(LibDir, "non_goal_6", "0.0.1", [stdlib,kernel,non_existant_goal], [], []), + + rlx_test_utils:create_app(LibDir, "goal_app_3", "0.0.1", [stdlib,kernel,goal_app_1,non_goal_2], [], [non_goal_3]), + rlx_test_utils:create_app(LibDir, "goal_app_4", "0.0.1", [stdlib,kernel,goal_app_1,non_goal_2], [], [non_existant_goal]), + rlx_test_utils:create_app(LibDir, "goal_app_5", "0.0.1", [stdlib,kernel,non_goal_6], [], [non_existant_goal]), + rlx_test_utils:create_app(LibDir, "goal_app_6", "0.0.1", [stdlib,kernel,non_existant_goal,goal_app_5], [], []), + [{lib_dir, LibDir} | Config]. @@ -144,6 +154,118 @@ make_release_keep_goal_order(Config) -> {non_goal_2,"0.0.1"}, {lib_dep_1,"0.0.1"}], AppSpecs). +optional_application(Config) -> + LibDir = ?config(lib_dir, Config), + OutputDir = ?config(out_dir, Config), + + RelxConfig = [{release, {optional_foo, "0.0.1"}, + [goal_app_3]}], + + {ok, State} = relx:build_release(optional_foo, [{root_dir, LibDir}, {lib_dirs, [LibDir]}, + {output_dir, OutputDir} | RelxConfig]), + + [{{optional_foo, "0.0.1"}, Release}] = maps:to_list(rlx_state:realized_releases(State)), + AppSpecs = rlx_release:app_specs(Release), + + %% goal_app_2 depends on goal_app_1, so the two are reordered when building the .rel file + ?assertMatch([{kernel,_}, + {stdlib,_}, + {lib_dep_1,"0.0.1",load}, + {non_goal_1,"0.0.1"}, + {goal_app_1,"0.0.1"}, + {non_goal_2,"0.0.1"}, + {non_goal_3,"0.0.1"}, + {goal_app_3,"0.0.1"}], AppSpecs). + +%% goal_app_4 has an optional application that doesn't exist +%% since it is optional the build won't fail +missing_optional_application(Config) -> + LibDir = ?config(lib_dir, Config), + OutputDir = ?config(out_dir, Config), + + RelxConfig = [{release, {missing_optional_foo, "0.0.1"}, + [goal_app_4]}], + + {ok, State} = relx:build_release(missing_optional_foo, [{root_dir, LibDir}, {lib_dirs, [LibDir]}, + {output_dir, OutputDir} | RelxConfig]), + + [{{missing_optional_foo, "0.0.1"}, Release}] = maps:to_list(rlx_state:realized_releases(State)), + AppSpecs = rlx_release:app_specs(Release), + + ?assertMatch([{kernel,_}, + {stdlib,_}, + {lib_dep_1,"0.0.1",load}, + {non_goal_1,"0.0.1"}, + {goal_app_1,"0.0.1"}, + {non_goal_2,"0.0.1"}, + {goal_app_4,"0.0.1"}], AppSpecs). + +duplicated_optional_applications(Config) -> + LibDir = ?config(lib_dir, Config), + OutputDir = ?config(out_dir, Config), + + RelxConfig = [{release, {duplicated_optional_foo, "0.0.1"}, + [goal_app_3]}], + + {ok, State} = relx:build_release(duplicated_optional_foo, [{root_dir, LibDir}, {lib_dirs, [LibDir]}, + {output_dir, OutputDir} | RelxConfig]), + + [{{duplicated_optional_foo, "0.0.1"}, Release}] = maps:to_list(rlx_state:realized_releases(State)), + AppSpecs = rlx_release:app_specs(Release), + + %% goal_app_2 depends on goal_app_1, so the two are reordered when building the .rel file + ?assertMatch([{kernel,_}, + {stdlib,_}, + {lib_dep_1,"0.0.1",load}, + {non_goal_1,"0.0.1"}, + {goal_app_1,"0.0.1"}, + {non_goal_2,"0.0.1"}, + {non_goal_3, "0.0.1"}, + {goal_app_3,"0.0.1"}], AppSpecs). + +missing_duplicated_optional_application(Config) -> + LibDir = ?config(lib_dir, Config), + OutputDir = ?config(out_dir, Config), + + RelxConfig = [{release, {missing_optional_foo, "0.0.1"}, + [goal_app_5]}], + + %% non_existant_goal is an optional app on goal_app_5 but a required dep of + %% non_goal_app_6 which causes the build to fail + ?assertError({error,{rlx_resolve,{app_not_found,non_existant_goal,undefined}}}, + relx:build_release(missing_optional_foo, [{root_dir, LibDir}, {lib_dirs, [LibDir]}, + {output_dir, OutputDir} | RelxConfig])), + + %% also test that if the app is required by the goal app but optional to + %% a dep of the goal it fails + RelxConfig1 = [{release, {missing_optional_foo, "0.0.1"}, + [goal_app_6]}], + ?assertError({error,{rlx_resolve,{app_not_found,non_existant_goal,undefined}}}, + relx:build_release(missing_optional_foo, [{root_dir, LibDir}, {lib_dirs, [LibDir]}, + {output_dir, OutputDir} | RelxConfig1 ])). + +excluded_optional_applications(Config) -> + LibDir = ?config(lib_dir, Config), + OutputDir = ?config(out_dir, Config), + + RelxConfig = [{release, {excluded_optional_foo, "0.0.1"}, + [goal_app_3]}, + {exclude_apps, [non_goal_2]}], + + {ok, State} = relx:build_release(excluded_optional_foo, [{root_dir, LibDir}, {lib_dirs, [LibDir]}, + {output_dir, OutputDir} | RelxConfig]), + + [{{excluded_optional_foo, "0.0.1"}, Release}] = maps:to_list(rlx_state:realized_releases(State)), + AppSpecs = rlx_release:app_specs(Release), + + ?assertMatch([{kernel,_}, + {stdlib,_}, + {lib_dep_1,"0.0.1",load}, + {non_goal_1,"0.0.1"}, + {goal_app_1,"0.0.1"}, + {non_goal_3, "0.0.1"}, + {goal_app_3,"0.0.1"}], AppSpecs). + make_config_release(Config) -> DataDir = ?config(priv_dir, Config), LibDir1 = ?config(lib_dir, Config), @@ -945,6 +1067,7 @@ make_exclude_modules_release(Config) -> {vsn,"0.0.1"}, {modules,[]}, {included_applications,_}, + {optional_applications,_}, {registered,_}, {applications,_}]}]}, file:consult(filename:join([OutputDir, "foo", "lib", diff --git a/test/rlx_test_utils.erl b/test/rlx_test_utils.erl index a144948ff..58d5b3fbf 100644 --- a/test/rlx_test_utils.erl +++ b/test/rlx_test_utils.erl @@ -5,25 +5,34 @@ -compile(export_all). create_app(Dir, Name, Vsn, Deps, LibDeps) -> + create_app(Dir, Name, Vsn, Deps, LibDeps, []). + +create_app(Dir, Name, Vsn, Deps, LibDeps, OptionalDeps) -> AppDir = filename:join([Dir, Name ++ "-" ++ Vsn]), - write_app_file(AppDir, Name, Vsn, app_modules(Name), Deps, LibDeps), + write_app_file(AppDir, Name, Vsn, app_modules(Name), Deps, LibDeps, OptionalDeps), write_src_file(AppDir, Name), write_priv_file(AppDir), compile_src_files(AppDir), - rlx_app_info:new(erlang:list_to_atom(Name), Vsn, AppDir, Deps, []). + rlx_app_info:new(erlang:list_to_atom(Name), Vsn, AppDir, Deps, LibDeps, OptionalDeps). create_full_app(Dir, Name, Vsn, Deps, LibDeps) -> + create_full_app(Dir, Name, Vsn, Deps, LibDeps, []). + +create_full_app(Dir, Name, Vsn, Deps, LibDeps, OptionalDeps) -> AppDir = filename:join([Dir, Name ++ "-" ++ Vsn]), - write_full_app_files(AppDir, Name, Vsn, Deps, LibDeps), + write_full_app_files(AppDir, Name, Vsn, Deps, LibDeps, OptionalDeps), compile_src_files(AppDir), rlx_app_info:new(erlang:list_to_atom(Name), Vsn, AppDir, - Deps, []). + Deps, LibDeps, OptionalDeps). create_empty_app(Dir, Name, Vsn, Deps, LibDeps) -> + create_empty_app(Dir, Name, Vsn, Deps, LibDeps, []). + +create_empty_app(Dir, Name, Vsn, Deps, LibDeps, OptionalDeps) -> AppDir = filename:join([Dir, Name ++ "-" ++ Vsn]), - write_app_file(AppDir, Name, Vsn, [], Deps, LibDeps), + write_app_file(AppDir, Name, Vsn, [], Deps, LibDeps, OptionalDeps), rlx_app_info:new(erlang:list_to_atom(Name), Vsn, AppDir, - Deps, []). + Deps, LibDeps, OptionalDeps). app_modules(Name) -> [list_to_atom(M ++ Name) || @@ -47,11 +56,11 @@ write_appup_file(AppInfo, DownVsn) -> ok = filelib:ensure_dir(Filename), ok = rlx_file_utils:write_term(Filename, {Vsn, [{DownVsn, []}], [{DownVsn, []}]}). -write_app_file(Dir, Name, Version, Modules, Deps, LibDeps) -> +write_app_file(Dir, Name, Version, Modules, Deps, LibDeps, OptionalDeps) -> Filename = filename:join([Dir, "ebin", Name ++ ".app"]), ok = filelib:ensure_dir(Filename), ok = rlx_file_utils:write_term(Filename, get_app_metadata(Name, Version, Modules, - Deps, LibDeps)). + Deps, LibDeps, OptionalDeps)). compile_src_files(Dir) -> %% compile all *.erl files in src to ebin @@ -70,21 +79,22 @@ compile_src_files(Dir) -> ok. -get_app_metadata(Name, Vsn, Modules, Deps, LibDeps) -> +get_app_metadata(Name, Vsn, Modules, Deps, LibDeps, OptionalDeps) -> {application, erlang:list_to_atom(Name), [{description, ""}, {vsn, Vsn}, {modules, Modules}, {included_applications, LibDeps}, + {optional_applications, OptionalDeps}, {registered, []}, {applications, Deps}]}. -write_full_app_files(Dir, Name, Vsn, Deps, LibDeps) -> +write_full_app_files(Dir, Name, Vsn, Deps, LibDeps, OptionalDeps) -> %% write out the .app file AppFilename = filename:join([Dir, "ebin", Name ++ ".app"]), ok = filelib:ensure_dir(AppFilename), ok = rlx_file_utils:write_term(AppFilename, - get_full_app_metadata(Name, Vsn, Deps, LibDeps)), + get_full_app_metadata(Name, Vsn, Deps, LibDeps, OptionalDeps)), %% write out the _app.erl file ApplicationFilename = filename:join([Dir, "src", Name ++ "_app.erl"]), ok = filelib:ensure_dir(ApplicationFilename), @@ -99,7 +109,7 @@ write_full_app_files(Dir, Name, Vsn, Deps, LibDeps) -> ok = file:write_file(GenServerFilename, gen_server_contents(Name)), ok. -get_full_app_metadata(Name, Vsn, Deps, LibDeps) -> +get_full_app_metadata(Name, Vsn, Deps, LibDeps, OptionalDeps) -> {application, erlang:list_to_atom(Name), [{description, ""}, {vsn, Vsn}, @@ -107,6 +117,7 @@ get_full_app_metadata(Name, Vsn, Deps, LibDeps) -> {mod, {erlang:list_to_atom(Name ++ "_app"), []}}, {included_applications, LibDeps}, + {optional_applications, OptionalDeps}, {registered, []}, {applications, Deps}]}.