Merge pull request #2697 from ferd/support-project-local-plugins

Add support for project-local plugins

.github/workflows/shelltests.yml

@@ -29,7 +29,7 @@ jobs:
     - name: Install and run shelltestrunner
       run: |
         sudo apt-get update
-        sudo apt-get install -y shelltestrunner build-essential
+        sudo apt-get install -y shelltestrunner build-essential cmake liblz4-dev
         cd rebar3_tests
         mix local.hex --force
         ./run_tests.sh

src/rebar.hrl

@@ -15,6 +15,7 @@
 -define(DEFAULT_BASE_DIR, "_build").
 -define(DEFAULT_ROOT_DIR, ".").
 -define(DEFAULT_PROJECT_APP_DIRS, ["apps/*", "lib/*", "."]).
+-define(DEFAULT_PROJECT_PLUGIN_DIRS, ["plugins/*"]).
 -define(DEFAULT_CHECKOUTS_DIR, "_checkouts").
 -define(DEFAULT_CHECKOUTS_OUT_DIR, "checkouts").
 -define(DEFAULT_DEPS_DIR, "lib").

src/rebar_app_discover.erl

@@ -64,8 +64,8 @@ do(State, LibDirs) ->
                                 OutDir = filename:join(DepsDir, Name),
                                 AppInfo2 = rebar_app_info:out_dir(AppInfo1, OutDir),
                                 ProjectDeps1 = lists:delete(Name, ProjectDeps),
-                                rebar_state:project_apps(StateAcc1
-                                                        ,rebar_app_info:deps(AppInfo2, ProjectDeps1));
+                                rebar_state:project_apps(StateAcc1,
+                                                         rebar_app_info:deps(AppInfo2, ProjectDeps1));
                             false ->
                                 ?INFO("Ignoring ~ts", [Name]),
                                 StateAcc

src/rebar_app_info.erl

@@ -638,7 +638,8 @@ valid(#app_info_t{valid=Valid}) ->
 
 %% @doc sets whether the app is valid (built) or not. If left unset,
 %% rebar3 will do the detection of the status itself.
--spec valid(t(), boolean()) -> t().
+%% Explicitly setting the value to `undefined' can force a re-evaluation.
+-spec valid(t(), boolean() | undefined) -> t().
 valid(AppInfo=#app_info_t{}, Valid) ->
     AppInfo#app_info_t{valid=Valid}.
 

src/rebar_dir.erl

@@ -13,6 +13,7 @@
          checkouts_out_dir/2,
          plugins_dir/1,
          lib_dirs/1,
+         project_plugin_dirs/1,
          home_dir/0,
          global_config_dir/1,
          global_config/1,
@@ -127,6 +128,12 @@ plugins_dir(State) ->
 lib_dirs(State) ->
     rebar_state:get(State, project_app_dirs, ?DEFAULT_PROJECT_APP_DIRS).
 
+%% @doc returns the list of relative path where the project plugins can
+%% be located.
+-spec project_plugin_dirs(rebar_state:t()) -> [file:filename_all()].
+project_plugin_dirs(State) ->
+    rebar_state:get(State, project_plugin_dirs, ?DEFAULT_PROJECT_PLUGIN_DIRS).
+
 %% @doc returns the user's home directory.
 -spec home_dir() -> file:filename_all().
 home_dir() ->

src/rebar_plugins.erl

@@ -8,7 +8,8 @@
         ,project_apps_install/1
         ,install/2
         ,handle_plugins/3
-        ,handle_plugins/4]).
+        ,handle_plugins/4
+        ,discover_plugins/1]).
 
 -include("rebar.hrl").
 
@@ -94,10 +95,11 @@ handle_plugins(Profile, Plugins, State, Upgrade) ->
     Locks = rebar_state:lock(State),
     DepsDir = rebar_state:get(State, deps_dir, ?DEFAULT_DEPS_DIR),
     State1 = rebar_state:set(State, deps_dir, ?DEFAULT_PLUGINS_DIR),
+    SrcPlugins = discover_plugins(Plugins, State),
     %% Install each plugin individually so if one fails to install it doesn't effect the others
     {_PluginProviders, State2} =
         lists:foldl(fun(Plugin, {PluginAcc, StateAcc}) ->
-                            {NewPlugins, NewState} = handle_plugin(Profile, Plugin, StateAcc, Upgrade),
+                            {NewPlugins, NewState} = handle_plugin(Profile, Plugin, StateAcc, SrcPlugins, Upgrade),
                             NewState1 = rebar_state:create_logic_providers(NewPlugins, NewState),
                             {PluginAcc++NewPlugins, NewState1}
                       end, {[], State1}, Plugins),
@@ -106,24 +108,34 @@ handle_plugins(Profile, Plugins, State, Upgrade) ->
     State3 = rebar_state:set(State2, deps_dir, DepsDir),
     rebar_state:lock(State3, Locks).
 
-handle_plugin(Profile, Plugin, State, Upgrade) ->
+handle_plugin(Profile, Plugin, State, SrcPlugins, Upgrade) ->
     try
-        {Apps, State2} = rebar_prv_install_deps:handle_deps_as_profile(Profile, State, [Plugin], Upgrade),
-        {no_cycle, Sorted} = rebar_prv_install_deps:find_cycles(Apps),
+        %% Inject top-level src plugins as project apps, so that they get skipped
+        %% by the installation as already seen
+        ProjectApps = rebar_state:project_apps(State),
+        State0 = rebar_state:project_apps(State, SrcPlugins),
+        %% We however have to pick the deps of top-level apps and promote them
+        %% directly to make sure they are installed if they were not also at the top level
+        TopDeps = top_level_deps(State, SrcPlugins),
+        %% Install the plugins
+        {Apps, State1} = rebar_prv_install_deps:handle_deps_as_profile(Profile, State0, [Plugin|TopDeps], Upgrade),
+        {no_cycle, Sorted} = rebar_prv_install_deps:find_cycles(SrcPlugins++Apps),
         ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []),
+        %% Return things to normal
+        State2 = rebar_state:project_apps(State1, ProjectApps),
 
         %% Add already built plugin deps to the code path
         ToBuildPaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild],
-        PreBuiltPaths = [Ebin || A <- Apps,
+        PreBuiltPaths = [Ebin || A <- Sorted,
                                  Ebin <- [rebar_app_info:ebin_dir(A)],
                                  not lists:member(Ebin, ToBuildPaths)],
         code:add_pathsa(PreBuiltPaths),
 
         %% Build plugin and its deps
-        build_plugins(ToBuild, Apps, State2),
+        build_plugins(ToBuild, Sorted, State2),
 
         %% Add newly built deps and plugin to code path
-        State3 = rebar_state:update_all_plugin_deps(State2, Apps),
+        State3 = rebar_state:update_all_plugin_deps(State2, Sorted),
         NewCodePaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild],
 
         %% Store plugin code paths so we can remove them when compiling project apps
@@ -172,3 +184,96 @@ validate_plugin(Plugin) ->
             end
     end.
 
+discover_plugins([], _) ->
+    %% don't search if nothing is declared
+    [];
+discover_plugins(_, State) ->
+    discover_plugins(State).
+
+discover_plugins(State) ->
+    %% only support this mode in an umbrella project to avoid cases where
+    %% this is used in a project intended to be an installed dependency and accidentally
+    %% relies on vendoring when not intended. Also skip for global plugins, this would
+    %% make no sense.
+    case lists:member(global, rebar_state:current_profiles(State)) orelse not is_umbrella(State) of
+        true ->
+            [];
+        false ->
+            %% Inject source paths for plugins to allow vendoring and umbrella
+            %% top-level declarations
+            BaseDir = rebar_state:dir(State),
+            LibDirs = rebar_dir:project_plugin_dirs(State),
+            Dirs = [filename:join(BaseDir, LibDir) || LibDir <- LibDirs],
+            RebarOpts = rebar_state:opts(State),
+            SrcDirs = rebar_dir:src_dirs(RebarOpts, ["src"]),
+            Found = rebar_app_discover:find_apps(Dirs, SrcDirs, all, State),
+            ?DEBUG("Found local plugins: ~p~n"
+                   "\tusing config: {project_plugin_dirs, ~p}",
+                   [[rebar_utils:to_atom(rebar_app_info:name(F)) || F <- Found],
+                    LibDirs]),
+            PluginsDir = rebar_dir:plugins_dir(State),
+            SetUp = lists:map(fun(App) ->
+                Name = rebar_app_info:name(App),
+                OutDir = filename:join(PluginsDir, Name),
+                prepare_plugin(rebar_app_info:out_dir(App, OutDir))
+            end, Found),
+            rebar_utils:sort_deps(SetUp)
+    end.
+
+is_umbrella(State) ->
+    %% We can't know if this is an umbrella project before running app discovery,
+    %% but plugins are installed before app discovery. So we do a heuristic.
+    %% The lib dirs we search contain things such as apps/, lib/, etc.
+    %% which contain sub-applications. Then there's a final search for the
+    %% local directory ("."), which finds the top-level app in a non-umbrella
+    %% project.
+    %%
+    %% So what we do here is look for the library directories without the ".",
+    %% and if none of these paths exist but one of the src_dirs exist, then
+    %% we know this is not an umbrella application.
+    Root = rebar_dir:root_dir(State),
+    LibPaths = lists:usort(rebar_dir:lib_dirs(State)) -- ["."],
+    SrcPaths = rebar_dir:src_dirs(rebar_state:opts(State), ["src"]),
+    lists:any(fun(Dir) -> [] == filelib:wildcard(filename:join(Root, Dir)) end, LibPaths)
+    andalso
+    lists:all(fun(Dir) -> not filelib:is_dir(filename:join(Root, Dir)) end, SrcPaths).
+
+prepare_plugin(AppInfo) ->
+    %% We need to handle plugins as dependencies to avoid re-building them
+    %% continuously. So here we copy the app directories to the dep location
+    %% and then change the AppInfo record to be redirected to the dep location.
+    AppDir = rebar_app_info:dir(AppInfo),
+    OutDir = rebar_app_info:out_dir(AppInfo),
+    rebar_prv_compile:copy_app_dirs(AppInfo, AppDir, OutDir),
+    Relocated = rebar_app_info:dir(AppInfo, OutDir),
+    case needs_rebuild(AppInfo) of
+        true -> rebar_app_info:valid(Relocated, false);     % force recompilation
+        false -> rebar_app_info:valid(Relocated, undefined) % force revalidation
+    end.
+
+top_level_deps(State, Apps) ->
+    CurrentProfiles = rebar_state:current_profiles(State),
+    Keys = lists:append([[{plugins, P}, {deps, P}] || P <- CurrentProfiles]),
+    RawDeps = lists:foldl(fun(App, Acc) ->
+        %% Only support the profiles we would with regular plugins?
+        lists:append([rebar_app_info:get(App, Key, []) || Key <- Keys]) ++ Acc
+    end, [], Apps),
+    rebar_utils:tup_dedup(RawDeps).
+
+needs_rebuild(AppInfo) ->
+    %% if source files are newer than built files then the code was edited
+    %% and can't be considered valid -- force a rebuild.
+    %%
+    %% we do this by reusing the compiler code for Erlang as a heuristic for
+    %% files to check. The actual compiler provider will do an in-depth
+    %% validation of each module that may or may not need recompiling.
+    #{src_dirs := SrcD, include_dirs := InclD,
+      out_mappings := List} = rebar_compiler_erl:context(AppInfo),
+    SrcDirs = SrcD++InclD,
+    OutDirs = [Dir || {_Ext, Dir} <- List],
+    newest_stamp(OutDirs) < newest_stamp(SrcDirs).
+
+newest_stamp(DirList) ->
+    lists:max([0] ++
+              [filelib:last_modified(F)
+               || F <- rebar_utils:find_files_in_dirs(DirList, ".+", true)]).

src/rebar_prv_compile.erl

@@ -7,6 +7,7 @@
          format_error/1]).
 
 -export([compile/2, compile/3, compile/4]).
+-export([copy_app_dirs/3]).
 
 -include_lib("providers/include/providers.hrl").
 -include("rebar.hrl").

src/rebar_prv_plugins_upgrade.erl

@@ -69,27 +69,37 @@ upgrade(Plugin, State) ->
         Dep ->
             Dep
     end,
-
+    LocalPlugins = [rebar_utils:to_atom(rebar_app_info:name(App))
+                    || App <- rebar_plugins:discover_plugins(State)],
     case Dep of
         not_found ->
             ?PRV_ERROR({not_found, Plugin});
         {ok, P, Profile} ->
-            State1 = rebar_state:set(State, deps_dir, ?DEFAULT_PLUGINS_DIR),
-            maybe_update_pkg(P, State1),
-            {Apps, State2} = rebar_prv_install_deps:handle_deps_as_profile(Profile, State1, [P], true),
+            case lists:member(P, LocalPlugins) of
+                true ->
+                    ?INFO("Plugin ~p is defined locally and does not need upgrading", [P]),
+                    {ok, State};
+                false ->
+                    do_upgrade(State, P, Profile)
+            end
+    end.
 
-            {no_cycle, Sorted} = rebar_prv_install_deps:find_cycles(Apps),
-            ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []),
+do_upgrade(State, P, Profile) ->
+    State1 = rebar_state:set(State, deps_dir, ?DEFAULT_PLUGINS_DIR),
+    maybe_update_pkg(P, State1),
+    {Apps, State2} = rebar_prv_install_deps:handle_deps_as_profile(Profile, State1, [P], true),
 
-            %% Add already built plugin deps to the code path
-            CodePaths = [rebar_app_info:ebin_dir(A) || A <- Apps -- ToBuild],
-            code:add_pathsa(CodePaths),
+    {no_cycle, Sorted} = rebar_prv_install_deps:find_cycles(Apps),
+    ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []),
 
-            %% Build plugin and its deps
-            _ = build_plugin(ToBuild, State2),
+    %% Add already built plugin deps to the code path
+    CodePaths = [rebar_app_info:ebin_dir(A) || A <- Apps -- ToBuild],
+    code:add_pathsa(CodePaths),
 
-            {ok, State}
-    end.
+    %% Build plugin and its deps
+    _ = build_plugin(ToBuild, State2),
+
+    {ok, State}.
 
 find_plugin(Plugin, Profiles, State) ->
     ec_lists:search(fun(Profile) ->