Merge pull request DiversityTemplating#3 from DiversityTemplating/cache

Files

src/component_handler.erl

@@ -25,46 +25,61 @@ handle(Req, State=#state{}) ->
 handle_get(Req, _State=#state{}) ->
     {PathInfo, Req2} = cowboy_req:path_info(Req),
     {ComponentName, Req3} = cowboy_req:binding(component, Req2),
-    Tags = git_utils:tags(ComponentName),
+    Tags = divapi_cache:get_tags(ComponentName),
     {ok, Req4} =
         case PathInfo of
             [] ->
                 cowboy_req:reply(200, ?JSON_HEADER, jiffy:encode(Tags), Req3);
-            [<<"*">>] ->
-                Tag = find_latest_tag(Tags),
-                Data = git_utils:get_diversity_json(ComponentName, Tag),
-                cowboy_req:reply(200, ?JSON_HEADER, Data, Req3);
-            [Tag] ->
-                Data = case lists:member(Tag, Tags) of
-                           true ->
-                               git_utils:get_diversity_json(ComponentName, Tag);
-                           false ->
-                               PatchNr = find_latest_patch(Tag, Tags),
-                               TagWithPatch = <<Tag/binary, ".", PatchNr/binary>>,
-                               %% Ensure that it's an valid tag
-                               true = lists:member(TagWithPatch, Tags),
-                               git_utils:get_diversity_json(ComponentName, TagWithPatch)
-                       end,
-                cowboy_req:reply(200, ?JSON_HEADER, Data, Req3);
-            [Tag, Settings]
-                when Settings =:= <<"settings">>;
-                     Settings =:= <<"settingsForm">> ->
-                Json = git_utils:get_diversity_json(ComponentName, Tag),
-                {DiversityData} = jiffy:decode(Json),
-                SettingsBinary = proplists:get_value(Settings, DiversityData, {[]}),
-                SettingsJson = jiffy:encode(SettingsBinary),
-                cowboy_req:reply(200, ?JSON_HEADER, SettingsJson, Req3);
+            [PartialTag | Routes] ->
+                Tag = expand_tag(PartialTag, Tags),
+                case Routes of
+                    [] ->
+                        Data = divapi_cache:get_diversity_json(ComponentName, Tag),
+                        case Data of
+                            undefined -> cowboy_req:reply(404, Req3);
+                            _ -> cowboy_req:reply(200, ?JSON_HEADER, Data, Req3)
+                        end;
+                    [Settings] when Settings =:= <<"settings">>;
+                                    Settings =:= <<"settingsForm">> ->
+                        Json = divapi_cache:get_diversity_json(ComponentName, Tag),
+                        {DiversityData} = jiffy:decode(Json),
+                        SettingsBinary = proplists:get_value(Settings, DiversityData, {[]}),
+                        SettingsJson = jiffy:encode(SettingsBinary),
+                        cowboy_req:reply(200, ?JSON_HEADER, SettingsJson, Req3);
+                    [<<"files">> | Path] ->
+                        File = filename:join(Path),
+                        FileData = git_utils:get_file(ComponentName, undefined, Tag, File),
+                        {Mime, Type, []} = cow_mimetypes:all(File),
+                        Header = [{<<"content-type">>, << Mime/binary, "/", Type/binary >>}],
+                        cowboy_req:reply(200, Header, FileData, Req3);
+                    _ ->
+                        cowboy_req:reply(404, Req3)
+
+                end;
             _ ->
                 cowboy_req:reply(404, Req3)
         end,
     {ok, Req4}.
 
+expand_tag(<<"*">>, Tags) -> find_latest_tag(Tags);
+expand_tag(Tag, Tags) ->
+    case lists:member(Tag, Tags) of
+        true ->
+            Tag;
+        false ->
+            PatchNr = find_latest_patch(Tag, Tags),
+            <<Tag/binary, ".", PatchNr/binary>>
+    end.
+
+
 handle_post(Req, _State=#state{}) ->
     {ComponentName, Req2} = cowboy_req:binding(component, Req),
     {PathInfo, Req3} = cowboy_req:path_info(Req2),
     case PathInfo of
         [<<"update">>] ->
             git_utils:git_refresh_repo(ComponentName),
+            divapi_cache:empty_cache(ComponentName),
+
             cowboy_req:reply(200, ?JSON_HEADER, <<"Updated">>, Req3);
         _ ->
             cowboy_req:reply(404, Req3)

src/component_list_handler.erl

@@ -20,10 +20,10 @@ handle(Req, State=#state{}) ->
             {QueryVal, Req3} = cowboy_req:qs_val(<<"grouping">>, Req2),
             Projects = case QueryVal of
                 undefined ->
-                    PublicProjects = gitlab_utils:get_public_projects(),
+                    PublicProjects = divapi_cache:get_components(),
                     get_components_information(PublicProjects);
                 Group when is_binary(Group) ->
-                    PublicProjects = gitlab_utils:get_public_projects(),
+                    PublicProjects = divapi_cache:get_components(),
                     filter_projects_by_grouping(PublicProjects, Group)
             end,
             cowboy_req:reply(200,
@@ -36,21 +36,21 @@ handle(Req, State=#state{}) ->
 	{ok, Req4, State}.
 
 %% @doc Returns a list with new maps only containing the keys in information_fields
--spec get_components_information([map()]) -> [map()].
+-spec get_components_information([binary()]) -> [map()].
 get_components_information(Components) ->
-    maps:fold(fun(ComponentName, ComponentUrl, Acc) ->
-        Json = git_utils:get_diversity_json(ComponentName, ComponentUrl, <<"HEAD">>),
-        DiversityMap = jiffy:decode(Json, [return_maps]),
+    lists:foldl(
+        fun({_,undefined}, Acc) -> Acc;
+           ({_,ComponentJson}, Acc) ->
+        DiversityMap =  jiffy:decode(ComponentJson, [return_maps]),
         Without = lists:filter(fun(K) -> not lists:member(K, ?INFORMATION_FIELDS) end, maps:keys(DiversityMap)),
         [maps:without(Without, DiversityMap) | Acc]
         end, [], Components).
 
 %% @doc Returns a list of component names filtered by given grouping
--spec filter_projects_by_grouping([map()], binary()) -> [binary()].
+-spec filter_projects_by_grouping([binary()], binary()) -> [binary()].
 filter_projects_by_grouping(Components, Grouping) ->
-    maps:fold(fun(ComponentName, ComponentUrl, Acc) ->
-        Json = git_utils:get_diversity_json(ComponentName, ComponentUrl, <<"HEAD">>),
-        DiversityMap = jiffy:decode(Json, [return_maps]),
+    lists:foldl(fun({ComponentName, ComponentJson},  Acc) ->
+        DiversityMap = jiffy:decode(ComponentJson, [return_maps]),
         Groups = maps:get(<<"grouping">>, DiversityMap, []),
         case lists:member(Grouping, Groups) of
             true -> [ComponentName | Acc];

src/divapi_app.erl

@@ -17,6 +17,7 @@ start(_Type, _Args) ->
     cowboy:start_http(diversity_api_listener, 100, [{port, ?PORT}],
         [{env, [{dispatch, Dispatch}]}]
     ),
+    divapi_cache:start_link(),
 	divapi_sup:start_link().
 
 stop(_State) ->

src/divapi_cache.erl

@@ -0,0 +1,134 @@
+-module(divapi_cache).
+-behaviour(gen_server).
+
+%% API.
+-export([start_link/0, get_components/0, get_tags/1, get_diversity_json/2, get_diversity_json/3, populate_cache/0, empty_cache/1]).
+%% gen_server.
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+    code_change/3, terminate/2]).
+
+-record(component, {name, tag, json}).
+
+-define(TIMEOUT, 800000).
+
+%% API.
+
+-spec start_link() -> {ok, pid()}.
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+populate_cache() ->
+    gen_server:call(?MODULE, populate).
+
+get_components() ->
+    gen_server:call(?MODULE, get_components, ?TIMEOUT).
+
+get_tags(Component) ->
+    gen_server:call(?MODULE, {get_tags, Component}).
+
+get_diversity_json(Component, Tag) ->
+    gen_server:call(?MODULE, {get_json, Component, Tag}).
+
+get_diversity_json(Component, ComponentUrl, Tag) ->
+    gen_server:call(?MODULE, {get_json, Component, ComponentUrl, Tag}).
+
+empty_cache(Component) ->
+    gen_server:call(?MODULE, {delete, Component}).
+
+
+%% gen_server.
+
+init([]) ->
+    Tab = ets:new(?MODULE, [bag, {keypos, #component.name}]),
+    {ok, Tab}.
+
+handle_call(populate, _From, Tab) ->
+    populate(Tab),
+    {reply, ok, Tab};
+
+handle_call(get_components, _From, Tab) ->
+    populate_if_empty(Tab),
+    Names = ets:match(Tab, #component{name='$1', tag= <<"HEAD">>, json='$2'}),
+    Reply = lists:usort(lists:map(fun list_to_tuple/1, Names)),
+    {reply, Reply, Tab};
+
+handle_call({get_tags, ComponentName}, _From, Tab) ->
+    Reply = case ets:lookup(Tab, ComponentName) of
+                [] ->
+                    case ets:first(Tab) of
+                        '$end_of_table' -> populate(Tab);
+                        _ -> add_component(ComponentName, undefined, Tab)
+                    end,
+                    Select = [{#component{name = ComponentName, tag = '$1', json = '_'}, [], ['$1']}],
+                    GitTags = ets:select(Tab, Select),
+                    lists:subtract(GitTags, [<<"HEAD">>]);
+                 ComponentRows ->
+                    [ Component#component.tag || Component <- ComponentRows, Component#component.tag /= <<"HEAD">> ]
+
+            end,
+    {reply, Reply, Tab};
+
+handle_call({get_json, ComponentName, Tag}, _From, Tab) ->
+    Reply = get_json_helper(ComponentName, undefined, Tag, Tab),
+    {reply, Reply, Tab};
+
+handle_call({get_json, ComponentName, ComponentUrl, Tag}, _From, Tab) ->
+    Reply = get_json_helper(ComponentName, ComponentUrl, Tag, Tab),
+    {reply, Reply, Tab};
+
+handle_call({delete, ComponentName}, _From, Tab) ->
+    ets:delete(Tab, ComponentName),
+    {reply, ok, Tab};
+
+handle_call(_Request, _From, State) ->
+    {reply, ignored, State}.
+
+populate_if_empty(Tab) ->
+    case ets:first(Tab) of
+        '$end_of_table' -> populate(Tab);
+        _ -> ok
+    end.
+
+populate(Tab) ->
+    Projects = gitlab_utils:get_public_projects(),
+    lists:foreach(fun({CName, CUrl}) -> add_component(CName, CUrl, Tab) end, maps:to_list(Projects)).
+
+add_component(ComponentName, ComponentUrl, Tab) ->
+    Tags = git_utils:tags(ComponentName, ComponentUrl),
+    lists:foreach(fun(Tag) -> insert_new_json(ComponentName, ComponentUrl, Tag, Tab) end, [<<"HEAD">> | Tags]).
+
+get_json_helper(ComponentName, ComponentUrl, Tag, Tab) ->
+    Select = [{#component{name = ComponentName, tag = Tag, json = '$1'}, [], ['$1']}],
+    case ets:select(Tab, Select) of
+        [] ->
+            case ets:first(Tab) of
+                '$end_of_table' -> populate(Tab);
+                _ -> add_component(ComponentName, ComponentUrl, Tab)
+            end,
+            insert_new_json(ComponentName, ComponentUrl, Tag, Tab);
+        [undefined] ->
+            ets:select_delete(Tab, Select),
+            insert_new_json(ComponentName, ComponentUrl, Tag, Tab);
+        [Json] ->
+            Json
+    end.
+
+insert_new_json(ComponentName, ComponentUrl, Tag, Tab) ->
+    NewJson = git_utils:get_diversity_json(ComponentName, ComponentUrl, Tag),
+    NewComponent = [#component{name = ComponentName, tag = Tag, json = NewJson}],
+    ets:insert(Tab, NewComponent),
+    NewJson.
+
+handle_cast(_Msg, State) ->
+    {noreply, State}.
+
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%% Internal

src/git_utils.erl

@@ -1,26 +1,19 @@
 -module(git_utils).
 
 %% API
--export([tags/1, get_diversity_json/2, get_diversity_json/3, git_refresh_repo/1]).
-
-
+-export([tags/2, get_diversity_json/3, git_refresh_repo/1, get_file/4]).
 
 %% @doc Returns a list with all tags in a git repository
--spec tags(binary()) -> [binary()].
-tags(RepoName) ->
+-spec tags(binary(), binary()) -> [binary()].
+tags(RepoName, RepoUrl) ->
     Cmd = "tag",
-    ResultString = get_git_result(RepoName, Cmd),
+    ResultString = get_git_result(RepoName, RepoUrl, Cmd),
     [list_to_binary(Tag) || Tag <- string:tokens(ResultString, "\n")].
 
 %% @doc Returns the diversity.json for a tag in a repo
--spec get_diversity_json(binary(), binary()) -> binary().
-get_diversity_json(RepoName, Tag) ->
-    Cmd = "show " ++ binary_to_list(Tag) ++ ":diversity.json",
-    list_to_binary(get_git_result(RepoName, Cmd)).
 -spec get_diversity_json(binary(), binary(), binary()) -> binary().
 get_diversity_json(RepoName, RepoUrl, Tag) ->
-    Cmd = "show " ++ binary_to_list(Tag) ++ ":diversity.json",
-    list_to_binary(get_git_result(RepoName, RepoUrl, Cmd)).
+    get_git_file(RepoName, RepoUrl, Tag, <<"diversity.json">>).
 
 %% @doc Fetches the latest tags for a repo
 -spec git_refresh_repo(binary()) -> any().
@@ -31,11 +24,20 @@ git_refresh_repo(RepoName) ->
     Cmd = "fetch origin master:master",
     git_cmd(Cmd).
 
+get_file(RepoName, RepoUrl, Tag, FilePath) ->
+    get_git_file(RepoName, RepoUrl, Tag, FilePath).
+
+
 %% ----------------------------------------------------------------------------
 %% Internal stuff
 %% ----------------------------------------------------------------------------
-get_git_result(RepoName, Cmd) ->
-    get_git_result(RepoName, undefined, Cmd).
+get_git_file(RepoName, RepoUrl, Tag, FilePath) ->
+    Cmd = "show " ++ binary_to_list(Tag) ++ ":" ++ binary_to_list(FilePath),
+    case get_git_result(RepoName, RepoUrl, Cmd) of
+        "fatal" ++ _ -> undefined;
+        Result -> list_to_binary(Result)
+    end.
+
 get_git_result(RepoName, RepoUrl, Cmd) ->
     {ok, RepoDir} = application:get_env(divapi, repo_dir),
     GitRepoName =  RepoDir ++ "/" ++ binary_to_list(RepoName) ++ ".git",