diff --git a/src/component_handler.erl b/src/component_handler.erl index bfba9d0..dda3a9d 100644 --- a/src/component_handler.erl +++ b/src/component_handler.erl @@ -25,46 +25,61 @@ handle(Req, State=#state{}) -> handle_get(Req, _State=#state{}) -> {PathInfo, Req2} = cowboy_req:path_info(Req), {ComponentName, Req3} = cowboy_req:binding(component, Req2), - Tags = git_utils:tags(ComponentName), + Tags = divapi_cache:get_tags(ComponentName), {ok, Req4} = case PathInfo of [] -> cowboy_req:reply(200, ?JSON_HEADER, jiffy:encode(Tags), Req3); - [<<"*">>] -> - Tag = find_latest_tag(Tags), - Data = git_utils:get_diversity_json(ComponentName, Tag), - cowboy_req:reply(200, ?JSON_HEADER, Data, Req3); - [Tag] -> - Data = case lists:member(Tag, Tags) of - true -> - git_utils:get_diversity_json(ComponentName, Tag); - false -> - PatchNr = find_latest_patch(Tag, Tags), - TagWithPatch = <>, - %% Ensure that it's an valid tag - true = lists:member(TagWithPatch, Tags), - git_utils:get_diversity_json(ComponentName, TagWithPatch) - end, - cowboy_req:reply(200, ?JSON_HEADER, Data, Req3); - [Tag, Settings] - when Settings =:= <<"settings">>; - Settings =:= <<"settingsForm">> -> - Json = git_utils:get_diversity_json(ComponentName, Tag), - {DiversityData} = jiffy:decode(Json), - SettingsBinary = proplists:get_value(Settings, DiversityData, {[]}), - SettingsJson = jiffy:encode(SettingsBinary), - cowboy_req:reply(200, ?JSON_HEADER, SettingsJson, Req3); + [PartialTag | Routes] -> + Tag = expand_tag(PartialTag, Tags), + case Routes of + [] -> + Data = divapi_cache:get_diversity_json(ComponentName, Tag), + case Data of + undefined -> cowboy_req:reply(404, Req3); + _ -> cowboy_req:reply(200, ?JSON_HEADER, Data, Req3) + end; + [Settings] when Settings =:= <<"settings">>; + Settings =:= <<"settingsForm">> -> + Json = divapi_cache:get_diversity_json(ComponentName, Tag), + {DiversityData} = jiffy:decode(Json), + SettingsBinary = proplists:get_value(Settings, DiversityData, {[]}), + SettingsJson = jiffy:encode(SettingsBinary), + cowboy_req:reply(200, ?JSON_HEADER, SettingsJson, Req3); + [<<"files">> | Path] -> + File = filename:join(Path), + FileData = git_utils:get_file(ComponentName, undefined, Tag, File), + {Mime, Type, []} = cow_mimetypes:all(File), + Header = [{<<"content-type">>, << Mime/binary, "/", Type/binary >>}], + cowboy_req:reply(200, Header, FileData, Req3); + _ -> + cowboy_req:reply(404, Req3) + + end; _ -> cowboy_req:reply(404, Req3) end, {ok, Req4}. +expand_tag(<<"*">>, Tags) -> find_latest_tag(Tags); +expand_tag(Tag, Tags) -> + case lists:member(Tag, Tags) of + true -> + Tag; + false -> + PatchNr = find_latest_patch(Tag, Tags), + <> + end. + + handle_post(Req, _State=#state{}) -> {ComponentName, Req2} = cowboy_req:binding(component, Req), {PathInfo, Req3} = cowboy_req:path_info(Req2), case PathInfo of [<<"update">>] -> git_utils:git_refresh_repo(ComponentName), + divapi_cache:empty_cache(ComponentName), + cowboy_req:reply(200, ?JSON_HEADER, <<"Updated">>, Req3); _ -> cowboy_req:reply(404, Req3) diff --git a/src/component_list_handler.erl b/src/component_list_handler.erl index 3672d72..3285af3 100644 --- a/src/component_list_handler.erl +++ b/src/component_list_handler.erl @@ -20,10 +20,10 @@ handle(Req, State=#state{}) -> {QueryVal, Req3} = cowboy_req:qs_val(<<"grouping">>, Req2), Projects = case QueryVal of undefined -> - PublicProjects = gitlab_utils:get_public_projects(), + PublicProjects = divapi_cache:get_components(), get_components_information(PublicProjects); Group when is_binary(Group) -> - PublicProjects = gitlab_utils:get_public_projects(), + PublicProjects = divapi_cache:get_components(), filter_projects_by_grouping(PublicProjects, Group) end, cowboy_req:reply(200, @@ -36,21 +36,21 @@ handle(Req, State=#state{}) -> {ok, Req4, State}. %% @doc Returns a list with new maps only containing the keys in information_fields --spec get_components_information([map()]) -> [map()]. +-spec get_components_information([binary()]) -> [map()]. get_components_information(Components) -> - maps:fold(fun(ComponentName, ComponentUrl, Acc) -> - Json = git_utils:get_diversity_json(ComponentName, ComponentUrl, <<"HEAD">>), - DiversityMap = jiffy:decode(Json, [return_maps]), + lists:foldl( + fun({_,undefined}, Acc) -> Acc; + ({_,ComponentJson}, Acc) -> + DiversityMap = jiffy:decode(ComponentJson, [return_maps]), Without = lists:filter(fun(K) -> not lists:member(K, ?INFORMATION_FIELDS) end, maps:keys(DiversityMap)), [maps:without(Without, DiversityMap) | Acc] end, [], Components). %% @doc Returns a list of component names filtered by given grouping --spec filter_projects_by_grouping([map()], binary()) -> [binary()]. +-spec filter_projects_by_grouping([binary()], binary()) -> [binary()]. filter_projects_by_grouping(Components, Grouping) -> - maps:fold(fun(ComponentName, ComponentUrl, Acc) -> - Json = git_utils:get_diversity_json(ComponentName, ComponentUrl, <<"HEAD">>), - DiversityMap = jiffy:decode(Json, [return_maps]), + lists:foldl(fun({ComponentName, ComponentJson}, Acc) -> + DiversityMap = jiffy:decode(ComponentJson, [return_maps]), Groups = maps:get(<<"grouping">>, DiversityMap, []), case lists:member(Grouping, Groups) of true -> [ComponentName | Acc]; diff --git a/src/divapi_app.erl b/src/divapi_app.erl index 3216b95..b255a78 100644 --- a/src/divapi_app.erl +++ b/src/divapi_app.erl @@ -17,6 +17,7 @@ start(_Type, _Args) -> cowboy:start_http(diversity_api_listener, 100, [{port, ?PORT}], [{env, [{dispatch, Dispatch}]}] ), + divapi_cache:start_link(), divapi_sup:start_link(). stop(_State) -> diff --git a/src/divapi_cache.erl b/src/divapi_cache.erl new file mode 100644 index 0000000..065f44a --- /dev/null +++ b/src/divapi_cache.erl @@ -0,0 +1,134 @@ +-module(divapi_cache). +-behaviour(gen_server). + +%% API. +-export([start_link/0, get_components/0, get_tags/1, get_diversity_json/2, get_diversity_json/3, populate_cache/0, empty_cache/1]). +%% gen_server. +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + code_change/3, terminate/2]). + +-record(component, {name, tag, json}). + +-define(TIMEOUT, 800000). + +%% API. + +-spec start_link() -> {ok, pid()}. +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +populate_cache() -> + gen_server:call(?MODULE, populate). + +get_components() -> + gen_server:call(?MODULE, get_components, ?TIMEOUT). + +get_tags(Component) -> + gen_server:call(?MODULE, {get_tags, Component}). + +get_diversity_json(Component, Tag) -> + gen_server:call(?MODULE, {get_json, Component, Tag}). + +get_diversity_json(Component, ComponentUrl, Tag) -> + gen_server:call(?MODULE, {get_json, Component, ComponentUrl, Tag}). + +empty_cache(Component) -> + gen_server:call(?MODULE, {delete, Component}). + + +%% gen_server. + +init([]) -> + Tab = ets:new(?MODULE, [bag, {keypos, #component.name}]), + {ok, Tab}. + +handle_call(populate, _From, Tab) -> + populate(Tab), + {reply, ok, Tab}; + +handle_call(get_components, _From, Tab) -> + populate_if_empty(Tab), + Names = ets:match(Tab, #component{name='$1', tag= <<"HEAD">>, json='$2'}), + Reply = lists:usort(lists:map(fun list_to_tuple/1, Names)), + {reply, Reply, Tab}; + +handle_call({get_tags, ComponentName}, _From, Tab) -> + Reply = case ets:lookup(Tab, ComponentName) of + [] -> + case ets:first(Tab) of + '$end_of_table' -> populate(Tab); + _ -> add_component(ComponentName, undefined, Tab) + end, + Select = [{#component{name = ComponentName, tag = '$1', json = '_'}, [], ['$1']}], + GitTags = ets:select(Tab, Select), + lists:subtract(GitTags, [<<"HEAD">>]); + ComponentRows -> + [ Component#component.tag || Component <- ComponentRows, Component#component.tag /= <<"HEAD">> ] + + end, + {reply, Reply, Tab}; + +handle_call({get_json, ComponentName, Tag}, _From, Tab) -> + Reply = get_json_helper(ComponentName, undefined, Tag, Tab), + {reply, Reply, Tab}; + +handle_call({get_json, ComponentName, ComponentUrl, Tag}, _From, Tab) -> + Reply = get_json_helper(ComponentName, ComponentUrl, Tag, Tab), + {reply, Reply, Tab}; + +handle_call({delete, ComponentName}, _From, Tab) -> + ets:delete(Tab, ComponentName), + {reply, ok, Tab}; + +handle_call(_Request, _From, State) -> + {reply, ignored, State}. + +populate_if_empty(Tab) -> + case ets:first(Tab) of + '$end_of_table' -> populate(Tab); + _ -> ok + end. + +populate(Tab) -> + Projects = gitlab_utils:get_public_projects(), + lists:foreach(fun({CName, CUrl}) -> add_component(CName, CUrl, Tab) end, maps:to_list(Projects)). + +add_component(ComponentName, ComponentUrl, Tab) -> + Tags = git_utils:tags(ComponentName, ComponentUrl), + lists:foreach(fun(Tag) -> insert_new_json(ComponentName, ComponentUrl, Tag, Tab) end, [<<"HEAD">> | Tags]). + +get_json_helper(ComponentName, ComponentUrl, Tag, Tab) -> + Select = [{#component{name = ComponentName, tag = Tag, json = '$1'}, [], ['$1']}], + case ets:select(Tab, Select) of + [] -> + case ets:first(Tab) of + '$end_of_table' -> populate(Tab); + _ -> add_component(ComponentName, ComponentUrl, Tab) + end, + insert_new_json(ComponentName, ComponentUrl, Tag, Tab); + [undefined] -> + ets:select_delete(Tab, Select), + insert_new_json(ComponentName, ComponentUrl, Tag, Tab); + [Json] -> + Json + end. + +insert_new_json(ComponentName, ComponentUrl, Tag, Tab) -> + NewJson = git_utils:get_diversity_json(ComponentName, ComponentUrl, Tag), + NewComponent = [#component{name = ComponentName, tag = Tag, json = NewJson}], + ets:insert(Tab, NewComponent), + NewJson. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%% Internal diff --git a/src/git_utils.erl b/src/git_utils.erl index eac2fa0..6bbcd13 100644 --- a/src/git_utils.erl +++ b/src/git_utils.erl @@ -1,26 +1,19 @@ -module(git_utils). %% API --export([tags/1, get_diversity_json/2, get_diversity_json/3, git_refresh_repo/1]). - - +-export([tags/2, get_diversity_json/3, git_refresh_repo/1, get_file/4]). %% @doc Returns a list with all tags in a git repository --spec tags(binary()) -> [binary()]. -tags(RepoName) -> +-spec tags(binary(), binary()) -> [binary()]. +tags(RepoName, RepoUrl) -> Cmd = "tag", - ResultString = get_git_result(RepoName, Cmd), + ResultString = get_git_result(RepoName, RepoUrl, Cmd), [list_to_binary(Tag) || Tag <- string:tokens(ResultString, "\n")]. %% @doc Returns the diversity.json for a tag in a repo --spec get_diversity_json(binary(), binary()) -> binary(). -get_diversity_json(RepoName, Tag) -> - Cmd = "show " ++ binary_to_list(Tag) ++ ":diversity.json", - list_to_binary(get_git_result(RepoName, Cmd)). -spec get_diversity_json(binary(), binary(), binary()) -> binary(). get_diversity_json(RepoName, RepoUrl, Tag) -> - Cmd = "show " ++ binary_to_list(Tag) ++ ":diversity.json", - list_to_binary(get_git_result(RepoName, RepoUrl, Cmd)). + get_git_file(RepoName, RepoUrl, Tag, <<"diversity.json">>). %% @doc Fetches the latest tags for a repo -spec git_refresh_repo(binary()) -> any(). @@ -31,11 +24,20 @@ git_refresh_repo(RepoName) -> Cmd = "fetch origin master:master", git_cmd(Cmd). +get_file(RepoName, RepoUrl, Tag, FilePath) -> + get_git_file(RepoName, RepoUrl, Tag, FilePath). + + %% ---------------------------------------------------------------------------- %% Internal stuff %% ---------------------------------------------------------------------------- -get_git_result(RepoName, Cmd) -> - get_git_result(RepoName, undefined, Cmd). +get_git_file(RepoName, RepoUrl, Tag, FilePath) -> + Cmd = "show " ++ binary_to_list(Tag) ++ ":" ++ binary_to_list(FilePath), + case get_git_result(RepoName, RepoUrl, Cmd) of + "fatal" ++ _ -> undefined; + Result -> list_to_binary(Result) + end. + get_git_result(RepoName, RepoUrl, Cmd) -> {ok, RepoDir} = application:get_env(divapi, repo_dir), GitRepoName = RepoDir ++ "/" ++ binary_to_list(RepoName) ++ ".git", diff --git a/test/component_list_handler_SUITE.erl b/test/component_list_handler_SUITE.erl new file mode 100644 index 0000000..19c12a9 --- /dev/null +++ b/test/component_list_handler_SUITE.erl @@ -0,0 +1,92 @@ +-module(component_list_handler_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-export([all/0]). + +%% API +-export([handle_grouping/1, handle_grouping_nomatch/1, handle_grouping_nomatch2/1]). + +all() -> [handle_grouping, handle_grouping_nomatch, handle_grouping_nomatch2]. + +-record(state, {}). + +handle_grouping(_Config) -> + SimplifiedReq = {http_req,<<"GET">>,'HTTP/1.1'}, + State = #state{}, + application:set_env(divapi, repo_dir, "gitdir"), + + meck:new(cowboy_req), + meck:expect(cowboy_req, reply, fun(200, _Headers, Body, _Req) -> {ok, Body} end), + meck:expect(cowboy_req, path, fun(Req) -> {<<"/components/">>,Req} end), + meck:expect(cowboy_req, qs_val, fun(<<"grouping">>, Req) -> {<<"sidebar">>, Req} end), + + meck:new(gitlab_utils), + + PublicProjects = maps:from_list([{<<"component">>, <<"url">>}]), + meck:expect(gitlab_utils, get_public_projects, fun() -> PublicProjects end), + + meck:new(git_utils), + GroupingJson = <<"{\"grouping\":[\"sidebar\",\"othergroup\"]}">>, + meck:expect(git_utils, get_diversity_json, fun(<<"component">>,_,_) -> GroupingJson end), + + Expected = <<"[\"component\"]">>, + Response = component_list_handler:handle(SimplifiedReq, State), + + Response = {ok, Expected, State}, + meck:unload(cowboy_req), + meck:unload(gitlab_utils), + meck:unload(git_utils). + +handle_grouping_nomatch(_Config) -> + SimplifiedReq = {http_req,<<"GET">>,'HTTP/1.1'}, + State = #state{}, + application:set_env(divapi, repo_dir, "gitdir"), + + meck:new(cowboy_req), + meck:expect(cowboy_req, reply, fun(200, _Headers, Body, _Req) -> {ok, Body} end), + meck:expect(cowboy_req, path, fun(Req) -> {<<"/components/">>,Req} end), + meck:expect(cowboy_req, qs_val, fun(<<"grouping">>, Req) -> {<<"nonexisting">>, Req} end), + + meck:new(gitlab_utils), + + PublicProjects = maps:from_list([{<<"component">>, <<"url">>}]), + meck:expect(gitlab_utils, get_public_projects, fun() -> PublicProjects end), + + meck:new(git_utils), + GroupingJson = <<"{\"grouping\":[\"sidebar\",\"othergroup\"]}">>, + meck:expect(git_utils, get_diversity_json, fun(<<"component">>,_,_) -> GroupingJson end), + + Expected = <<"[]">>, + Response = component_list_handler:handle(SimplifiedReq, State), + + Response = {ok, Expected, State}, + meck:unload(cowboy_req), + meck:unload(gitlab_utils), + meck:unload(git_utils). + +handle_grouping_nomatch2(_Config) -> + SimplifiedReq = {http_req,<<"GET">>,'HTTP/1.1'}, + State = #state{}, + application:set_env(divapi, repo_dir, "gitdir"), + + meck:new(cowboy_req), + meck:expect(cowboy_req, reply, fun(200, _Headers, Body, _Req) -> {ok, Body} end), + meck:expect(cowboy_req, path, fun(Req) -> {<<"/components/">>,Req} end), + meck:expect(cowboy_req, qs_val, fun(<<"grouping">>, Req) -> {<<"sidebar">>, Req} end), + + meck:new(gitlab_utils), + + PublicProjects = maps:from_list([{<<"component">>, <<"url">>}]), + meck:expect(gitlab_utils, get_public_projects, fun() -> PublicProjects end), + + meck:new(git_utils), + GroupingJson = <<"{\"grouping\":[\"othergroup\"]}">>, + meck:expect(git_utils, get_diversity_json, fun(<<"component">>,_,_) -> GroupingJson end), + + Expected = <<"[]">>, + Response = component_list_handler:handle(SimplifiedReq, State), + + Response = {ok, Expected, State}, + meck:unload(cowboy_req), + meck:unload(gitlab_utils), + meck:unload(git_utils). \ No newline at end of file diff --git a/test/divapi_cache_SUITE.erl b/test/divapi_cache_SUITE.erl new file mode 100644 index 0000000..7ca9a7c --- /dev/null +++ b/test/divapi_cache_SUITE.erl @@ -0,0 +1,11 @@ +-module(divapi_cache_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-export([all/0]). +%% API +-export([test_empty_cache/1]). + +all() -> [test_empty_cache]. + +test_empty_cache(_Config) -> +ok.