Added reindex command (#522)

* Added reindex command

Added a reindex code lens so we can rebuild indexes without resorting
to manual operations.

As we build the indexing infrastructure out, we'll likely need to
rebuild the index a lot. Presently, this means restarting the server,
which can take some time, so I thought i'd add a command to do it
instead. I tried using code actions, but this was a bit fraught, so
instead, I used a code lens on the project definition in your mix.exs file.
The code lens disappears when clicked and reappears when the indexing job is done.

apps/proto/.formatter.exs

@@ -8,6 +8,7 @@ proto_dsl = [
   defrequest: 2,
   defresponse: 1,
   deftype: 1,
+  server_request: 2,
   server_request: 3
 ]
 

apps/proto/lib/lexical/proto.ex

@@ -10,7 +10,10 @@ defmodule Lexical.Proto do
       import Proto.Alias, only: [defalias: 1]
       import Proto.Enum, only: [defenum: 1]
       import Proto.Notification, only: [defnotification: 1, defnotification: 2]
-      import Proto.Request, only: [defrequest: 1, defrequest: 2, server_request: 3]
+
+      import Proto.Request,
+        only: [defrequest: 1, defrequest: 2, server_request: 2, server_request: 3]
+
       import Proto.Response, only: [defresponse: 1]
       import Proto.Type, only: [deftype: 1]
     end

apps/proto/lib/lexical/proto/request.ex

@@ -26,6 +26,16 @@ defmodule Lexical.Proto.Request do
     end
   end
 
+  defmacro server_request(method, response_module_ast) do
+    quote do
+      unquote(do_defrequest(method, [], __CALLER__))
+
+      def parse_response(response) do
+        unquote(response_module_ast).parse(response)
+      end
+    end
+  end
+
   defp fetch_types(params_module_ast, env) do
     params_module =
       params_module_ast
@@ -94,11 +104,17 @@ defmodule Lexical.Proto.Request do
 
       defimpl Jason.Encoder, for: unquote(lsp_module_name) do
         def encode(request, opts) do
+          params =
+            case Map.take(request, unquote(param_names)) do
+              empty when map_size(empty) == 0 -> nil
+              params -> params
+            end
+
           %{
             id: request.id,
             jsonrpc: "2.0",
             method: unquote(method),
-            params: Map.take(request, unquote(param_names))
+            params: params
           }
           |> Jason.Encode.map(opts)
         end

apps/protocol/lib/generated/lexical/protocol/types/code_lens.ex

@@ -0,0 +1,7 @@
+# This file's contents are auto-generated. Do not edit.
+defmodule Lexical.Protocol.Types.CodeLens do
+  alias Lexical.Proto
+  alias Lexical.Protocol.Types
+  use Proto
+  deftype command: optional(Types.Command), data: optional(any()), range: Types.Range
+end

apps/protocol/lib/generated/lexical/protocol/types/code_lens/params.ex

@@ -0,0 +1,10 @@
+# This file's contents are auto-generated. Do not edit.
+defmodule Lexical.Protocol.Types.CodeLens.Params do
+  alias Lexical.Proto
+  alias Lexical.Protocol.Types
+  use Proto
+
+  deftype partial_result_token: optional(Types.Progress.Token),
+          text_document: Types.TextDocument.Identifier,
+          work_done_token: optional(Types.Progress.Token)
+end

apps/protocol/lib/generated/lexical/protocol/types/execute_command/params.ex

@@ -0,0 +1,10 @@
+# This file's contents are auto-generated. Do not edit.
+defmodule Lexical.Protocol.Types.ExecuteCommand.Params do
+  alias Lexical.Proto
+  alias Lexical.Protocol.Types
+  use Proto
+
+  deftype arguments: optional(list_of(any())),
+          command: string(),
+          work_done_token: optional(Types.Progress.Token)
+end

apps/protocol/lib/generated/lexical/protocol/types/execute_command/registration/options.ex

@@ -0,0 +1,6 @@
+# This file's contents are auto-generated. Do not edit.
+defmodule Lexical.Protocol.Types.ExecuteCommand.Registration.Options do
+  alias Lexical.Proto
+  use Proto
+  deftype commands: list_of(string()), work_done_progress: optional(boolean())
+end

apps/protocol/lib/lexical/protocol/requests.ex

@@ -52,6 +52,12 @@ defmodule Lexical.Protocol.Requests do
     defrequest "textDocument/codeAction", Types.CodeAction.Params
   end
 
+  defmodule CodeLens do
+    use Proto
+
+    defrequest "textDocument/codeLens", Types.CodeLens.Params
+  end
+
   defmodule Completion do
     use Proto
 
@@ -64,6 +70,12 @@ defmodule Lexical.Protocol.Requests do
     defrequest "textDocument/hover", Types.Hover.Params
   end
 
+  defmodule ExecuteCommand do
+    use Proto
+
+    defrequest "workspace/executeCommand", Types.ExecuteCommand.Params
+  end
+
   # Server -> Client requests
 
   defmodule RegisterCapability do
@@ -80,5 +92,11 @@ defmodule Lexical.Protocol.Requests do
                    Responses.ShowMessage
   end
 
+  defmodule CodeLensRefresh do
+    use Proto
+
+    server_request "workspace/codeLens/refresh", Responses.Empty
+  end
+
   use Proto, decoders: :requests
 end

apps/protocol/lib/lexical/protocol/responses.ex

@@ -39,6 +39,11 @@ defmodule Lexical.Protocol.Responses do
     defresponse optional(list_of(Types.CodeAction))
   end
 
+  defmodule CodeLens do
+    use Proto
+    defresponse optional(list_of(Types.CodeLens))
+  end
+
   defmodule Completion do
     use Proto
 
@@ -57,6 +62,12 @@ defmodule Lexical.Protocol.Responses do
     defresponse optional(Types.Hover)
   end
 
+  defmodule ExecuteCommand do
+    use Proto
+
+    defresponse optional(any())
+  end
+
   # Client -> Server responses
 
   defmodule ShowMessage do

apps/remote_control/lib/lexical/remote_control/api.ex

@@ -9,6 +9,7 @@ defmodule Lexical.RemoteControl.Api do
   alias Lexical.RemoteControl.CodeAction
   alias Lexical.RemoteControl.CodeIntelligence
   alias Lexical.RemoteControl.CodeMod
+  alias Lexical.RemoteControl.Commands
 
   require Logger
 
@@ -113,4 +114,12 @@ defmodule Lexical.RemoteControl.Api do
   def broadcast(%Project{} = project, message) do
     RemoteControl.call(project, RemoteControl.Dispatch, :broadcast, [message])
   end
+
+  def reindex(%Project{} = project) do
+    RemoteControl.call(project, Commands.Reindex, :perform, [project])
+  end
+
+  def index_running?(%Project{} = project) do
+    RemoteControl.call(project, Commands.Reindex, :running?, [])
+  end
 end

apps/remote_control/lib/lexical/remote_control/api/messages.ex

@@ -39,6 +39,10 @@ defmodule Lexical.RemoteControl.Api.Messages do
 
   defrecord :struct_discovered, module: nil, fields: []
 
+  defrecord :project_reindex_requested, project: nil
+
+  defrecord :project_reindexed, project: nil, elapsed_ms: 0, status: :success
+
   @type compile_status :: :successful | :error
   @type name_and_arity :: {atom, non_neg_integer}
   @type field_list :: Keyword.t() | [atom]
@@ -118,4 +122,14 @@ defmodule Lexical.RemoteControl.Api.Messages do
           )
 
   @type struct_discovered :: record(:struct_discovered, module: module(), fields: field_list())
+
+  @type project_reindex_requested ::
+          record(:project_reindex_requested, project: Lexical.Project.t())
+
+  @type project_reindexed ::
+          record(:project_reindexed,
+            project: Lexical.Project.t(),
+            elapsed_ms: non_neg_integer(),
+            status: :success | {:error, term()}
+          )
 end

apps/remote_control/lib/lexical/remote_control/application.ex

@@ -16,6 +16,7 @@ defmodule Lexical.RemoteControl.Application do
     children =
       if RemoteControl.project_node?() do
         [
+          {RemoteControl.Commands.Reindex, nil},
           RemoteControl.Module.Loader,
           {RemoteControl.Dispatch, progress: true},
           RemoteControl.ModuleMappings,

apps/remote_control/lib/lexical/remote_control/commands/reindex.ex

@@ -0,0 +1,149 @@
+defmodule Lexical.RemoteControl.Commands.Reindex do
+  defmodule State do
+    alias Lexical.Ast.Analysis
+    alias Lexical.Document
+    alias Lexical.RemoteControl.Search
+    alias Lexical.RemoteControl.Search.Indexer
+
+    require Logger
+    defstruct reindex_fun: nil, index_task: nil, pending_updates: %{}
+
+    def new(reindex_fun) do
+      %__MODULE__{reindex_fun: reindex_fun}
+    end
+
+    def set_task(%__MODULE__{} = state, {_, _} = task) do
+      %__MODULE__{state | index_task: task}
+    end
+
+    def clear_task(%__MODULE__{} = state) do
+      %__MODULE__{state | index_task: nil}
+    end
+
+    def reindex_uri(%__MODULE__{index_task: nil} = state, uri) do
+      case entries_for_uri(uri) do
+        {:ok, path, entries} ->
+          Search.Store.update(path, entries)
+
+        _ ->
+          :ok
+      end
+
+      state
+    end
+
+    def reindex_uri(%__MODULE__{} = state, uri) do
+      case entries_for_uri(uri) do
+        {:ok, path, entries} ->
+          put_in(state.pending_updates[path], entries)
+
+        _ ->
+          state
+      end
+    end
+
+    def flush_pending_updates(%__MODULE__{} = state) do
+      Enum.each(state.pending_updates, fn {path, entries} ->
+        Search.Store.update(path, entries)
+      end)
+
+      %__MODULE__{state | pending_updates: %{}}
+    end
+
+    defp entries_for_uri(uri) do
+      with {:ok, %Document{} = document, %Analysis{} = analysis} <-
+             Document.Store.fetch(uri, :analysis),
+           {:ok, entries} <- Indexer.Quoted.index(analysis) do
+        {:ok, document.path, entries}
+      else
+        error ->
+          Logger.error("Could not update index because #{inspect(error)}")
+          error
+      end
+    end
+  end
+
+  @moduledoc """
+  A simple genserver that prevents more than one reindexing job from running at the same time
+  """
+
+  alias Lexical.Document
+  alias Lexical.Project
+  alias Lexical.RemoteControl.Api
+  alias Lexical.RemoteControl.Dispatch
+  alias Lexical.RemoteControl.Search
+
+  use GenServer
+  import Api.Messages
+
+  def start_link(reindex_fun) when is_function(reindex_fun, 1) do
+    GenServer.start_link(__MODULE__, reindex_fun, name: __MODULE__)
+  end
+
+  def start_link(_) do
+    start_link(&do_reindex/1)
+  end
+
+  def uri(uri) do
+    GenServer.cast(__MODULE__, {:reindex_uri, uri})
+  end
+
+  def perform(%Project{} = project) do
+    GenServer.call(__MODULE__, {:perform, project})
+  end
+
+  def running? do
+    GenServer.call(__MODULE__, :running?)
+  end
+
+  @impl GenServer
+  def init(reindex_fun) do
+    {:ok, State.new(reindex_fun)}
+  end
+
+  @impl GenServer
+  def handle_call(:running?, _from, %State{index_task: index_task} = state) do
+    {:reply, match?({_, _}, index_task), state}
+  end
+
+  def handle_call({:perform, project}, _from, %State{index_task: nil} = state) do
+    index_task = spawn_monitor(fn -> state.reindex_fun.(project) end)
+    {:reply, :ok, State.set_task(state, index_task)}
+  end
+
+  def handle_call({:perform, _project}, _from, state) do
+    {:reply, {:error, "Already Running"}, state}
+  end
+
+  @impl GenServer
+  def handle_cast({:reindex_uri, uri}, %State{} = state) do
+    {:noreply, State.reindex_uri(state, uri)}
+  end
+
+  @impl GenServer
+  def handle_info({:DOWN, ref, :process, pid, _reason}, %State{index_task: {pid, ref}} = state) do
+    new_state =
+      state
+      |> State.flush_pending_updates()
+      |> State.clear_task()
+
+    {:noreply, new_state}
+  end
+
+  defp do_reindex(%Project{} = project) do
+    Dispatch.broadcast(project_reindex_requested(project: project))
+
+    {elapsed_us, result} =
+      :timer.tc(fn ->
+        with {:ok, entries} <- Search.Indexer.create_index(project) do
+          Search.Store.replace(entries)
+        end
+      end)
+
+    Dispatch.broadcast(
+      project_reindexed(project: project, elapsed_ms: round(elapsed_us / 1000), status: :success)
+    )
+
+    result
+  end
+end

apps/remote_control/lib/lexical/remote_control/dispatch/handlers/indexing.ex

@@ -1,10 +1,9 @@
 defmodule Lexical.RemoteControl.Dispatch.Handlers.Indexing do
-  alias Lexical.Ast.Analysis
   alias Lexical.Document
   alias Lexical.RemoteControl.Api.Messages
+  alias Lexical.RemoteControl.Commands
   alias Lexical.RemoteControl.Dispatch
   alias Lexical.RemoteControl.Search
-  alias Lexical.RemoteControl.Search.Indexer
 
   require Logger
   import Messages
@@ -26,11 +25,7 @@ defmodule Lexical.RemoteControl.Dispatch.Handlers.Indexing do
   end
 
   defp reindex(uri) do
-    with {:ok, %Document{} = document, %Analysis{} = analysis} <-
-           Document.Store.fetch(uri, :analysis),
-         {:ok, entries} <- Indexer.Quoted.index(analysis) do
-      Search.Store.update(document.path, entries)
-    end
+    Commands.Reindex.uri(uri)
   end
 
   def delete_path(uri) do

apps/remote_control/lib/lexical/remote_control/search/indexer.ex

@@ -107,7 +107,7 @@ defmodule Lexical.RemoteControl.Search.Indexer do
         fn chunk ->
           block_bytes = chunk |> Enum.map(&Map.get(path_to_size_map, &1)) |> Enum.sum()
           result = Enum.map(chunk, processor)
-          update_progress.(block_bytes, nil)
+          update_progress.(block_bytes, "Indexing")
           result
         end,
         timeout: timeout