From 9037d696e6b0635832f025b53bcb5196519f6607 Mon Sep 17 00:00:00 2001 From: JINNOUCHI Yasushi Date: Sun, 27 Aug 2023 18:51:16 +0900 Subject: [PATCH] feat: add logic to store data by native code (#130) * refactor: make logic for Database be abstract * feat: add logic for DB by string.dump * fix: run with async.void to run synchronously * test: add tests for native feature * feat!: sort candidates by path when score is same This is needed because candidates from SQLite is sorted by id, but ones from native is sorted by path. * chore: clean up types * feat: add lock/unlock feature to access DB * test: use async version of busted And disable benchmark tests (fix later) * test: add tests for file_lock * chore: use more explicit names * chore: use plenary.log instead * fix: wait async functions definitely * feat: add migrator * chore: fix logging * fix: detect emptiness of the table * fix: deal with buffer with no names * test: loosen the condition temporarily * test: add tests for migrator * fix: return true when the table is not empty * feat: load sqlite lazily not to require in start * chore: add logging to calculate time for fetching * feat: add converter from native code to SQLite * feat: warn when sqlite.lua is not available * feat: add FrecencyMigrateDB to migrate DB * docs: add note for native code logic * test: ignore type bug --- .luarc.json | 1 + README.md | 49 ++- lua/frecency/database.lua | 141 ++----- lua/frecency/database/native.lua | 173 +++++++++ lua/frecency/database/sqlite.lua | 135 +++++++ lua/frecency/entry_maker.lua | 5 +- lua/frecency/file_lock.lua | 84 +++++ lua/frecency/frecency.lua | 77 +++- lua/frecency/migrator.lua | 83 +++++ lua/frecency/picker.lua | 25 +- lua/frecency/sqlite.lua | 17 + lua/frecency/tests/file_lock_spec.lua | 135 +++++++ lua/frecency/tests/frecency_spec.lua | 494 +++++++++++++------------ lua/frecency/tests/migrator_spec.lua | 129 +++++++ lua/frecency/tests/util.lua | 22 +- lua/frecency/types.lua | 50 ++- lua/frecency/wait.lua | 54 +++ lua/telescope/_extensions/frecency.lua | 5 +- 18 files changed, 1268 insertions(+), 411 deletions(-) create mode 100644 lua/frecency/database/native.lua create mode 100644 lua/frecency/database/sqlite.lua create mode 100644 lua/frecency/file_lock.lua create mode 100644 lua/frecency/migrator.lua create mode 100644 lua/frecency/sqlite.lua create mode 100644 lua/frecency/tests/file_lock_spec.lua create mode 100644 lua/frecency/tests/migrator_spec.lua create mode 100644 lua/frecency/wait.lua diff --git a/.luarc.json b/.luarc.json index 51b643d7..b6ed4233 100644 --- a/.luarc.json +++ b/.luarc.json @@ -1,6 +1,7 @@ { "diagnostics": { "globals": [ + "a", "describe", "it", "vim" diff --git a/README.md b/README.md index dfc5d717..77d8e140 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The score is calculated using the age of the 10 most recent timestamps and the t ### Score calculation -``` +```lua score = frequency * recency_score / max_number_of_timestamps ``` ## What about files that are neither 'frequent' _or_ 'recent' ? @@ -59,9 +59,11 @@ If the active buffer (prior to the finder being launched) is attached to an LSP ## Requirements - [telescope.nvim](https://github.com/nvim-telescope/telescope.nvim) (required) -- [sqlite.lua](https://github.com/kkharji/sqlite.lua) (required) +- [sqlite.lua][] (required) - [nvim-web-devicons](https://github.com/kyazdani42/nvim-web-devicons) (optional) +[sqlite.lua]: https://github.com/kkharji/sqlite.lua + Timestamps and file records are stored in an [SQLite3](https://www.sqlite.org/index.html) database for persistence and speed. This plugin uses `sqlite.lua` to perform the database transactions. @@ -73,9 +75,9 @@ This plugin uses `sqlite.lua` to perform the database transactions. use { "nvim-telescope/telescope-frecency.nvim", config = function() - require"telescope".load_extension("frecency") + require("telescope").load_extension "frecency" end, - requires = {"kkharji/sqlite.lua"} + requires = { "kkharji/sqlite.lua" }, } ``` @@ -85,9 +87,9 @@ use { { "nvim-telescope/telescope-frecency.nvim", config = function() - require"telescope".load_extension("frecency") + require("telescope").load_extension "frecency" end, - dependencies = {"kkharji/sqlite.lua"} + dependencies = { "kkharji/sqlite.lua" }, } ``` @@ -102,7 +104,7 @@ If no database is found when running Neovim with the plugin installed, a new one or to map to a key: ```lua -vim.api.nvim_set_keymap("n", "", "lua require('telescope').extensions.frecency.frecency()", {noremap = true, silent = true}) +vim.api.nvim_set_keymap("n", "", "Telescope frecency") ``` Use a specific workspace tag: @@ -114,7 +116,7 @@ Use a specific workspace tag: or ```lua -vim.api.nvim_set_keymap("n", "", "lua require('telescope').extensions.frecency.frecency({ workspace = 'CWD' })", {noremap = true, silent = true}) +vim.api.nvim_set_keymap("n", "", "Telescope frecency workspace=CWD") ``` Filter tags are applied by typing the `:tag:` name (adding surrounding colons) in the finder query. @@ -124,7 +126,7 @@ Entering `:` will trigger omnicompletion for available tags. See [default configuration](https://github.com/nvim-telescope/telescope.nvim#telescope-defaults) for full details on configuring Telescope. -- `db_root` (default: `nil`) +- `db_root` (default: `vim.fn.stdpath "data"`) Path to parent directory of custom database location. Defaults to `$XDG_DATA_HOME/nvim` if unset. @@ -133,7 +135,7 @@ See [default configuration](https://github.com/nvim-telescope/telescope.nvim#tel Default workspace tag to filter by e.g. `'CWD'` to filter by default to the current directory. Can be overridden at query time by specifying another filter like `':*:'`. -- `ignore_patterns` (default: `{"*.git/*", "*/tmp/*"}`) +- `ignore_patterns` (default: `{ "*.git/*", "*/tmp/*", "term://*" }`) Patterns in this table control which files are indexed (and subsequently which you'll see in the finder results). @@ -141,7 +143,7 @@ See [default configuration](https://github.com/nvim-telescope/telescope.nvim#tel To see the scores generated by the algorithm in the results, set this to `true`. -- `workspaces` (default: {}) +- `workspaces` (default: `{}`) This table contains mappings of `workspace_tag` -> `workspace_directory` The key corresponds to the `:tag_name` used to select the filter in queries. @@ -164,10 +166,13 @@ See [default configuration](https://github.com/nvim-telescope/telescope.nvim#tel show_filter_column = { "LSP", "CWD", "FOO" } ``` +- `use_sqlite` (default: `true`) ***experimental feature*** + + Use [sqlite.lua] `true` or native code `false`. See [*Remove dependency for sqlite.lua*](#remove-dependency-for-sqlite.lua) for the detail. ### Example Configuration: -``` +```lua telescope.setup { extensions = { frecency = { @@ -187,12 +192,14 @@ telescope.setup { } ``` -### SQL database location +## Note for Database -The default location for the sqlite3 database is `$XDG_DATA_HOME/nvim` (eg `~/.local/share/nvim/` on linux). +### Location + +The default location for the database is `$XDG_DATA_HOME/nvim` (eg `~/.local/share/nvim/` on linux). This can be configured with the `db_root` config option. -### SQL database maintainance +### Maintainance By default, frecency will prune files that no longer exist from the database. In certain workflows, switching branches in a repository, that behaviour might not be desired. @@ -210,7 +217,17 @@ The command `FrecencyValidate` can be used to clean the database when `auto_vali :FrecencyValidate! ``` -### Highlight Groups +### Remove dependency for [sqlite.lua][] + +***This is an experimental feature.*** + +In default, it uses SQLite3 library to access the DB. When `use_sqlite` option is set to `false`, it stores the whole data and saves them with encoding by `string.dump()` Lua function. + +With this, we can remove the dependency for [sqlite.lua][] and obtain faster speed to open `:Telescope frecency`. + +You can migrate from SQLite DB into native code by `:FrecencyMigrateDB` command. It converts data into native code, but does not delete the existent SQLite DB. You can use old SQLite logic by `use_sqlite = true` again. + +## Highlight Groups ```vim TelescopeBufferLoaded diff --git a/lua/frecency/database.lua b/lua/frecency/database.lua index a2aa7cad..35565ad0 100644 --- a/lua/frecency/database.lua +++ b/lua/frecency/database.lua @@ -1,136 +1,43 @@ -local sqlite = require "sqlite" -local log = require "plenary.log" - +---@diagnostic disable: missing-return, unused-local ---@class FrecencyDatabaseConfig ---@field root string ----@class FrecencySqlite: sqlite_db ----@field files sqlite_tbl ----@field timestamps sqlite_tbl - ----@class FrecencyFile ----@field count integer ----@field id integer ----@field path string ----@field score integer calculated from count and age - ----@class FrecencyTimestamp ----@field age integer calculated from timestamp ----@field file_id integer ----@field id integer ----@field timestamp number - ---@class FrecencyDatabaseGetFilesOptions ---@field path string? ---@field workspace string? ---@class FrecencyDatabase ---@field config FrecencyDatabaseConfig ----@field private buf_registered_flag_name string ----@field private fs FrecencyFS ----@field private sqlite FrecencySqlite +---@field has_entry fun(): boolean +---@field new fun(fs: FrecencyFS, config: FrecencyDatabaseConfig): FrecencyDatabase +---@field protected fs FrecencyFS local Database = {} ----@param fs FrecencyFS ----@param config FrecencyDatabaseConfig ----@return FrecencyDatabase -Database.new = function(fs, config) - local lib = sqlite.lib --[[@as sqlite_lib]] - local self = setmetatable( - { config = config, buf_registered_flag_name = "telescope_frecency_registered", fs = fs }, - { __index = Database } - ) - self.sqlite = sqlite { - uri = self.config.root .. "/file_frecency.sqlite3", - files = { id = true, count = { "integer", default = 1, required = true }, path = "string" }, - timestamps = { - id = true, - file_id = { "integer", reference = "files.id", on_delete = "cascade" }, - timestamp = { "real", default = lib.julianday "now" }, - }, - } - return self -end - ----@return boolean -function Database:has_entry() - return self.sqlite.files:count() > 0 -end - ---@param paths string[] ----@return integer -function Database:insert_files(paths) - if #paths == 0 then - return 0 - end - ---@param path string - return self.sqlite.files:insert(vim.tbl_map(function(path) - return { path = path, count = 0 } -- TODO: remove when sql.nvim#97 is closed - end, paths)) -end +---@return nil +function Database:insert_files(paths) end ----@param workspace string? ----@return FrecencyFile[] -function Database:get_files(workspace) - local query = workspace and { contains = { path = { workspace .. "/*" } } } or {} - log.debug { query = query } - return self.sqlite.files:get(query) -end +---@return integer[]|string[] +function Database:unlinked_entries() end ----@param datetime string? ISO8601 format string ----@return FrecencyTimestamp[] -function Database:get_timestamps(datetime) - local lib = sqlite.lib - local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer") - return self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } } -end +---@param files integer[]|string[] +---@return nil +function Database:remove_files(files) end ---@param path string ----@return integer: id of the file entry ----@return boolean: whether the entry is inserted (true) or updated (false) -function Database:upsert_files(path) - local file = self.sqlite.files:get({ where = { path = path } })[1] --[[@as FrecencyFile?]] - if file then - self.sqlite.files:update { where = { id = file.id }, set = { count = file.count + 1 } } - return file.id, false - end - return self.sqlite.files:insert { path = path }, true -end - ----@param file_id integer ----@param datetime string? ISO8601 format string ----@return integer -function Database:insert_timestamps(file_id, datetime) - return self.sqlite.timestamps:insert { - file_id = file_id, - timestamp = datetime and sqlite.lib.julianday(datetime) or nil, - } -end - ----@param file_id integer ---@param max_count integer -function Database:trim_timestamps(file_id, max_count) - local timestamps = self.sqlite.timestamps:get { where = { file_id = file_id } } --[[@as FrecencyTimestamp[] ]] - local trim_at = timestamps[#timestamps - max_count + 1] - if trim_at then - self.sqlite.timestamps:remove { file_id = tostring(file_id), id = "<" .. tostring(trim_at.id) } - end -end - ----@return integer[] -function Database:unlinked_entries() - ---@param file FrecencyFile - return self.sqlite.files:map(function(file) - if not self.fs:is_valid_path(file.path) then - return file.id - end - end) -end - ----@param ids integer[] +---@param datetime string? ---@return nil -function Database:remove_files(ids) - self.sqlite.files:remove { id = ids } -end +function Database:update(path, max_count, datetime) end -return Database +---@async +---@class FrecencyDatabaseEntry +---@field ages number[] +---@field count integer +---@field path string +---@field score number + +---@param workspace string? +---@param datetime string? +---@return FrecencyDatabaseEntry[] +function Database:get_entries(workspace, datetime) end diff --git a/lua/frecency/database/native.lua b/lua/frecency/database/native.lua new file mode 100644 index 00000000..6600e60a --- /dev/null +++ b/lua/frecency/database/native.lua @@ -0,0 +1,173 @@ +local FileLock = require "frecency.file_lock" +local wait = require "frecency.wait" +local log = require "plenary.log" +local async = require "plenary.async" --[[@as PlenaryAsync]] + +---@class FrecencyDatabaseNative: FrecencyDatabase +---@field version "v1" +---@field filename string +---@field file_lock FrecencyFileLock +---@field table FrecencyDatabaseNativeTable +local Native = {} + +---@class FrecencyDatabaseNativeTable +---@field version string +---@field records table + +---@class FrecencyDatabaseNativeRecord +---@field count integer +---@field timestamps integer[] + +---@param fs FrecencyFS +---@param config FrecencyDatabaseConfig +---@return FrecencyDatabaseNative +Native.new = function(fs, config) + local version = "v1" + local self = setmetatable({ + config = config, + fs = fs, + table = { version = version, records = {} }, + version = version, + }, { __index = Native }) + self.filename = self.config.root .. "/file_frecency.bin" + self.file_lock = FileLock.new(self.filename) + wait(function() + self:load() + end) + return self +end + +---@return boolean +function Native:has_entry() + return not vim.tbl_isempty(self.table.records) +end + +---@param paths string[] +---@return nil +function Native:insert_files(paths) + if #paths == 0 then + return + end + for _, path in ipairs(paths) do + self.table.records[path] = { count = 1, timestamps = { 0 } } + end + wait(function() + self:save() + end) +end + +---@return string[] +function Native:unlinked_entries() + local paths = {} + for file in pairs(self.table.records) do + if not self.fs:is_valid_path(file) then + table.insert(paths, file) + end + end + return paths +end + +---@param paths string[] +function Native:remove_files(paths) + for _, file in ipairs(paths) do + self.table.records[file] = nil + end + wait(function() + self:save() + end) +end + +---@param path string +---@param max_count integer +---@param datetime string? +function Native:update(path, max_count, datetime) + local record = self.table.records[path] or { count = 0, timestamps = {} } + record.count = record.count + 1 + local now = self:now(datetime) + table.insert(record.timestamps, now) + if #record.timestamps > max_count then + local new_table = {} + for i = #record.timestamps - max_count + 1, #record.timestamps do + table.insert(new_table, record.timestamps[i]) + end + record.timestamps = new_table + end + self.table.records[path] = record + wait(function() + self:save() + end) +end + +---@param workspace string? +---@param datetime string? +---@return FrecencyDatabaseEntry[] +function Native:get_entries(workspace, datetime) + -- TODO: check mtime of DB and reload it + -- self:load() + local now = self:now(datetime) + local items = {} + for path, record in pairs(self.table.records) do + if not workspace or path:find(workspace .. "/", 1, true) then + table.insert(items, { + path = path, + count = record.count, + ages = vim.tbl_map(function(v) + return (now - v) / 60 + end, record.timestamps), + }) + end + end + return items +end + +---@private +---@param datetime string? +---@return integer +function Native:now(datetime) + return datetime and vim.fn.strptime("%FT%T%z", datetime) or os.time() +end + +---@async +---@return nil +function Native:load() + local start = os.clock() + local err, data = self.file_lock:with(function() + local err, st = async.uv.fs_stat(self.filename) + if err then + return nil + end + local fd + err, fd = async.uv.fs_open(self.filename, "r", tonumber("644", 8)) + assert(not err) + local data + err, data = async.uv.fs_read(fd, st.size) + assert(not err) + assert(not async.uv.fs_close(fd)) + return data + end) + assert(not err, err) + local tbl = loadstring(data or "")() --[[@as FrecencyDatabaseNativeTable?]] + if tbl and tbl.version == self.version then + self.table = tbl + end + log.debug(("load() takes %f seconds"):format(os.clock() - start)) +end + +---@async +---@return nil +function Native:save() + local start = os.clock() + local err = self.file_lock:with(function() + local f = assert(load("return " .. vim.inspect(self.table))) + local data = string.dump(f) + local err, fd = async.uv.fs_open(self.filename, "w", tonumber("644", 8)) + assert(not err) + assert(not async.uv.fs_write(fd, data)) + assert(not async.uv.fs_close(fd)) + return nil + end) + assert(not err, err) + log.debug(("save() takes %f seconds"):format(os.clock() - start)) +end + +return Native diff --git a/lua/frecency/database/sqlite.lua b/lua/frecency/database/sqlite.lua new file mode 100644 index 00000000..4940ea70 --- /dev/null +++ b/lua/frecency/database/sqlite.lua @@ -0,0 +1,135 @@ +local sqlite = require "frecency.sqlite" +local log = require "plenary.log" + +---@class FrecencySqliteDB: sqlite_db +---@field files sqlite_tbl +---@field timestamps sqlite_tbl + +---@class FrecencyFile +---@field count integer +---@field id integer +---@field path string +---@field score integer calculated from count and age + +---@class FrecencyTimestamp +---@field age integer calculated from timestamp +---@field file_id integer +---@field id integer +---@field timestamp number + +---@class FrecencyDatabaseSqlite: FrecencyDatabase +---@field sqlite FrecencySqliteDB +local Sqlite = {} + +---@param fs FrecencyFS +---@param config FrecencyDatabaseConfig +---@return FrecencyDatabaseSqlite +Sqlite.new = function(fs, config) + local lib = sqlite.lib + local self = setmetatable( + { config = config, buf_registered_flag_name = "telescope_frecency_registered", fs = fs }, + { __index = Sqlite } + ) + self.sqlite = sqlite { + uri = self.config.root .. "/file_frecency.sqlite3", + files = { id = true, count = { "integer", default = 1, required = true }, path = "string" }, + timestamps = { + id = true, + file_id = { "integer", reference = "files.id", on_delete = "cascade" }, + timestamp = { "real", default = lib.julianday "now" }, + }, + } + return self +end + +---@return boolean +function Sqlite:has_entry() + return self.sqlite.files:count() > 0 +end + +---@param paths string[] +---@return integer +function Sqlite:insert_files(paths) + if #paths == 0 then + return 0 + end + ---@param path string + return self.sqlite.files:insert(vim.tbl_map(function(path) + return { path = path, count = 0 } -- TODO: remove when sql.nvim#97 is closed + end, paths)) +end + +---@param workspace string? +---@param datetime string? +---@return FrecencyDatabaseEntry[] +function Sqlite:get_entries(workspace, datetime) + local query = workspace and { contains = { path = { workspace .. "/*" } } } or {} + log.debug { query = query } + local files = self.sqlite.files:get(query) --[[@as FrecencyFile[] ]] + local lib = sqlite.lib + local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer") + local timestamps = self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } } --[[@as FrecencyTimestamp[] ]] + ---@type table + local age_map = {} + for _, timestamp in ipairs(timestamps) do + if not age_map[timestamp.file_id] then + age_map[timestamp.file_id] = {} + end + table.insert(age_map[timestamp.file_id], timestamp.age) + end + local items = {} + for _, file in ipairs(files) do + table.insert(items, { path = file.path, count = file.count, ages = age_map[file.id] }) + end + return items +end + +---@param datetime string? ISO8601 format string +---@return FrecencyTimestamp[] +function Sqlite:get_timestamps(datetime) + local lib = sqlite.lib + local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer") + return self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } } +end + +---@param path string +---@param count integer +---@param datetime string? +---@return nil +function Sqlite:update(path, count, datetime) + local file = self.sqlite.files:get({ where = { path = path } })[1] --[[@as FrecencyFile?]] + local file_id + if file then + self.sqlite.files:update { where = { id = file.id }, set = { count = file.count + 1 } } + file_id = file.id + else + file_id = self.sqlite.files:insert { path = path } + end + self.sqlite.timestamps:insert { + file_id = file_id, + timestamp = datetime and sqlite.lib.julianday(datetime) or nil, + } + local timestamps = self.sqlite.timestamps:get { where = { file_id = file_id } } --[[@as FrecencyTimestamp[] ]] + local trim_at = timestamps[#timestamps - count + 1] + if trim_at then + self.sqlite.timestamps:remove { file_id = tostring(file_id), id = "<" .. tostring(trim_at.id) } + end +end + +---@return integer[] +function Sqlite:unlinked_entries() + ---@param file FrecencyFile + return self.sqlite.files:map(function(file) + if not self.fs:is_valid_path(file.path) then + return file.id + end + end) +end + +---@param ids integer[] +---@return nil +function Sqlite:remove_files(ids) + self.sqlite.files:remove { id = ids } +end + +return Sqlite diff --git a/lua/frecency/entry_maker.lua b/lua/frecency/entry_maker.lua index 01461739..89d3b1e0 100644 --- a/lua/frecency/entry_maker.lua +++ b/lua/frecency/entry_maker.lua @@ -24,7 +24,10 @@ EntryMaker.new = function(fs, web_devicons, config) end, vim.api.nvim_list_bufs()) self.loaded = {} for _, bufnr in ipairs(loaded_bufnrs) do - self.loaded[vim.api.nvim_buf_get_name(bufnr)] = true + local bufname = vim.api.nvim_buf_get_name(bufnr) + if bufname then + self.loaded[bufname] = true + end end return self end diff --git a/lua/frecency/file_lock.lua b/lua/frecency/file_lock.lua new file mode 100644 index 00000000..ed4cb053 --- /dev/null +++ b/lua/frecency/file_lock.lua @@ -0,0 +1,84 @@ +local async = require "plenary.async" --[[@as PlenaryAsync]] +local log = require "plenary.log" + +---@class FrecencyFileLock +---@field base string +---@field config FrecencyFileLockConfig +---@field filename string +local FileLock = {} + +---@class FrecencyFileLockConfig +---@field retry integer default: 5 +---@field interval integer default: 500 + +---@param path string +---@param opts FrecencyFileLockConfig? +---@return FrecencyFileLock +FileLock.new = function(path, opts) + local config = vim.tbl_extend("force", { retry = 5, interval = 500 }, opts or {}) + local self = setmetatable({ config = config }, { __index = FileLock }) + self.filename = path .. ".lock" + return self +end + +---@async +---@return string? err +function FileLock:get() + local count = 0 + local err, fd + while true do + count = count + 1 + err, fd = async.uv.fs_open(self.filename, "wx", tonumber("600", 8)) + if not err then + break + end + async.util.sleep(self.config.interval) + if count == self.config.retry then + log.debug(("file_lock get() failed: retry count reached: %d"):format(count)) + return "failed to get lock" + end + log.debug(("file_lock get() retry: %d"):format(count)) + end + err = async.uv.fs_close(fd) + if err then + log.debug("file_lock get() failed: " .. err) + return err + end +end + +---@async +---@return string? err +function FileLock:release() + local err = async.uv.fs_stat(self.filename) + if err then + log.debug("file_lock release() not found: " .. err) + return "lock not found" + end + err = async.uv.fs_unlink(self.filename) + if err then + log.debug("file_lock release() unlink failed: " .. err) + return err + end +end + +---@async +---@generic T +---@param f fun(): T +---@return string? err +---@return T +function FileLock:with(f) + local err = self:get() + if err then + return err, nil + end + local ok, result_or_err = pcall(f) + err = self:release() + if err then + return err, nil + elseif ok then + return nil, result_or_err + end + return result_or_err, nil +end + +return FileLock diff --git a/lua/frecency/frecency.lua b/lua/frecency/frecency.lua index 76fe3411..eb9028f1 100644 --- a/lua/frecency/frecency.lua +++ b/lua/frecency/frecency.lua @@ -1,10 +1,13 @@ -local Database = require "frecency.database" +local Sqlite = require "frecency.database.sqlite" +local Native = require "frecency.database.native" local EntryMaker = require "frecency.entry_maker" local FS = require "frecency.fs" local Finder = require "frecency.finder" +local Migrator = require "frecency.migrator" local Picker = require "frecency.picker" local Recency = require "frecency.recency" local WebDevicons = require "frecency.web_devicons" +local sqlite_module = require "frecency.sqlite" local log = require "plenary.log" ---@class Frecency @@ -13,6 +16,7 @@ local log = require "plenary.log" ---@field private database FrecencyDatabase ---@field private finder FrecencyFinder ---@field private fs FrecencyFS +---@field private migrator FrecencyMigrator ---@field private picker FrecencyPicker ---@field private recency FrecencyRecency local Frecency = {} @@ -29,6 +33,7 @@ local Frecency = {} ---@field show_filter_column boolean|string[]|nil default: true ---@field show_scores boolean? default: false ---@field show_unindexed boolean? default: true +---@field use_sqlite boolean? default: true ---@field workspaces table? default: {} ---@param opts FrecencyConfig? @@ -47,10 +52,21 @@ Frecency.new = function(opts) show_filter_column = true, show_scores = false, show_unindexed = true, + use_sqlite = true, workspaces = {}, }, opts or {}) local self = setmetatable({ buf_registered = {}, config = config }, { __index = Frecency })--[[@as Frecency]] self.fs = FS.new { ignore_patterns = config.ignore_patterns } + + local Database + if not self.config.use_sqlite then + Database = Native + elseif not sqlite_module.can_use then + self:warn "use_sqlite = true, but sqlite module can not be found. It fallbacks to native code." + Database = Native + else + Database = Sqlite + end self.database = Database.new(self.fs, { root = config.db_root }) local web_devicons = WebDevicons.new(not config.disable_devicons) local entry_maker = EntryMaker.new(self.fs, web_devicons, { @@ -59,6 +75,7 @@ Frecency.new = function(opts) }) self.finder = Finder.new(entry_maker, self.fs) self.recency = Recency.new() + self.migrator = Migrator.new(self.fs, self.recency, self.config.db_root) return self end @@ -84,6 +101,10 @@ function Frecency:setup() self:validate_database() end + vim.api.nvim_create_user_command("FrecencyMigrateDB", function() + self:migrate_database() + end, { desc = "Migrate DB telescope-frecency to native code" }) + local group = vim.api.nvim_create_augroup("TelescopeFrecency", {}) vim.api.nvim_create_autocmd({ "BufWinEnter", "BufWritePost" }, { desc = "Update database for telescope-frecency", @@ -152,7 +173,6 @@ function Frecency:validate_database(force) end) end ----@private ---@param bufnr integer ---@param datetime string? ISO8601 format string function Frecency:register(bufnr, datetime) @@ -160,15 +180,40 @@ function Frecency:register(bufnr, datetime) if self.buf_registered[bufnr] or not self.fs:is_valid_path(path) then return end - local id, inserted = self.database:upsert_files(path) - self.database:insert_timestamps(id, datetime) - self.database:trim_timestamps(id, self.recency.config.max_count) - if inserted and self.picker then - self.picker:discard_results() - end + self.database:update(path, self.recency.config.max_count, datetime) self.buf_registered[bufnr] = true end +---@param to_sqlite boolean? +---@return nil +function Frecency:migrate_database(to_sqlite) + local prompt = to_sqlite and "migrate the DB into SQLite from native code?" + or "migrate the DB into native code from SQLite?" + vim.ui.select({ "y", "n" }, { + prompt = prompt, + ---@param item "y"|"n" + ---@return string + format_item = function(item) + return item == "y" and "Yes, Migrate it." or "No. Do nothing." + end, + }, function(item) + if item == "n" then + self:notify "migration aborted" + return + elseif to_sqlite then + if sqlite_module.can_use then + self.migrator:to_sqlite() + else + self:error "sqlite.lua is unavailable" + return + end + else + self.migrator:to_v1() + end + self:notify "migration finished successfully" + end) +end + ---@private ---@param fmt string ---@param ... any? @@ -185,4 +230,20 @@ function Frecency:notify(fmt, ...) vim.notify(self:message(fmt, ...)) end +---@private +---@param fmt string +---@param ... any? +---@return nil +function Frecency:warn(fmt, ...) + vim.notify(self:message(fmt, ...), vim.log.levels.WARN) +end + +---@private +---@param fmt string +---@param ... any? +---@return nil +function Frecency:error(fmt, ...) + vim.notify(self:message(fmt, ...), vim.log.levels.ERROR) +end + return Frecency diff --git a/lua/frecency/migrator.lua b/lua/frecency/migrator.lua new file mode 100644 index 00000000..90c213c0 --- /dev/null +++ b/lua/frecency/migrator.lua @@ -0,0 +1,83 @@ +local Sqlite = require "frecency.database.sqlite" +local Native = require "frecency.database.native" +local wait = require "frecency.wait" + +---@class FrecencyMigrator +---@field fs FrecencyFS +---@field recency FrecencyRecency +---@field root string +local Migrator = {} + +---@param fs FrecencyFS +---@param recency FrecencyRecency +---@param root string +---@return FrecencyMigrator +Migrator.new = function(fs, recency, root) + return setmetatable({ fs = fs, recency = recency, root = root }, { __index = Migrator }) +end + +---@return nil +function Migrator:to_v1() + local native = Native.new(self.fs, { root = self.root }) + native.table = self:v1_table_from_sqlite() + wait(function() + native:save() + end) +end + +---@return nil +function Migrator:to_sqlite() + local sqlite = Sqlite.new(self.fs, { root = self.root }) + local native = Native.new(self.fs, { root = self.root }) + for path, record in pairs(native.table.records) do + local file_id = sqlite.sqlite.files:insert { path = path, count = record.count } + sqlite.sqlite.timestamps:insert(vim.tbl_map(function(timestamp) + return { file_id = file_id, timestamp = ('julianday(datetime(%d, "unixepoch"))'):format(timestamp) } + end, record.timestamps)) + end +end + +---@private +---@return FrecencyDatabaseNativeTable +function Migrator:v1_table_from_sqlite() + local sqlite = Sqlite.new(self.fs, { root = self.root }) + ---@type FrecencyDatabaseNativeTable + local tbl = { version = "v1", records = {} } + local files = sqlite.sqlite.files:get {} --[[@as FrecencyFile[] ]] + ---@type table + local path_map = {} + for _, file in ipairs(files) do + tbl.records[file.path] = { count = file.count, timestamps = { 0 } } + path_map[file.id] = file.path + end + -- local timestamps = sqlite.sqlite.timestamps:get { keys = { "id", "file_id", epoch = "unixepoch(timestamp)" } } --[[@as FrecencyTimestamp[] ]] + local timestamps = sqlite.sqlite.timestamps:get { + keys = { "id", "file_id", epoch = "cast(strftime('%s', timestamp) as integer)" }, + } --[[@as FrecencyTimestamp[] ]] + table.sort(timestamps, function(a, b) + return a.id < b.id + end) + for _, timestamp in ipairs(timestamps) do + local path = path_map[timestamp.file_id] + if path then + local record = tbl.records[path] + if record then + if #record.timestamps == 1 and record.timestamps[1] == 0 then + record.timestamps = {} + end + ---@diagnostic disable-next-line: undefined-field + table.insert(record.timestamps, timestamp.epoch) + if #record.timestamps > self.recency.config.max_count then + local new_table = {} + for i = #record.timestamps - self.recency.config.max_count + 1, #record.timestamps do + table.insert(new_table, record.timestamps[i]) + end + record.timestamps = new_table + end + end + end + end + return tbl +end + +return Migrator diff --git a/lua/frecency/picker.lua b/lua/frecency/picker.lua index 97e8520a..eaebded2 100644 --- a/lua/frecency/picker.lua +++ b/lua/frecency/picker.lua @@ -184,28 +184,15 @@ end ---@return FrecencyFile[] function Picker:fetch_results(workspace, datetime) log.debug { workspace = workspace or "NONE" } - local start_files = os.clock() - local files = self.database:get_files(workspace) - log.debug { files = #files } - log.debug(("it takes %f seconds in fetching files with workspace: %s"):format(os.clock() - start_files, workspace)) - local start_timesatmps = os.clock() - local timestamps = self.database:get_timestamps(datetime) - log.debug { timestamps = #timestamps } - log.debug(("it takes %f seconds in fetching all timestamps"):format(os.clock() - start_timesatmps)) + local start_fetch = os.clock() + local files = self.database:get_entries(workspace, datetime) + log.debug(("it takes %f seconds in fetching entries"):format(os.clock() - start_fetch)) local start_results = os.clock() local elapsed_recency = 0 - ---@type table - local age_map = {} - for _, timestamp in ipairs(timestamps) do - if not age_map[timestamp.file_id] then - age_map[timestamp.file_id] = {} - end - table.insert(age_map[timestamp.file_id], timestamp.age) - end for _, file in ipairs(files) do local start_recency = os.clock() - local ages = age_map[file.id] --[[@as number[]?]] - file.score = ages and self.recency:calculate(file.count, ages) or 0 + file.score = file.ages and self.recency:calculate(file.count, file.ages) or 0 + file.ages = nil elapsed_recency = elapsed_recency + (os.clock() - start_recency) end log.debug(("it takes %f seconds in calculating recency"):format(elapsed_recency)) @@ -213,7 +200,7 @@ function Picker:fetch_results(workspace, datetime) local start_sort = os.clock() table.sort(files, function(a, b) - return a.score > b.score + return a.score > b.score or (a.score == b.score and a.path > b.path) end) log.debug(("it takes %f seconds in sorting"):format(os.clock() - start_sort)) return files diff --git a/lua/frecency/sqlite.lua b/lua/frecency/sqlite.lua new file mode 100644 index 00000000..80ad1234 --- /dev/null +++ b/lua/frecency/sqlite.lua @@ -0,0 +1,17 @@ +---@class FrecencySqlite +---@field can_use boolean +---@field lib sqlite_lib +---@overload fun(opts: table): FrecencySqliteDB + +return setmetatable({}, { + __index = function(_, k) + if k == "lib" then + return require("sqlite").lib + elseif k == "can_use" then + return not not vim.F.npcall(require, "sqlite") + end + end, + __call = function(_, opts) + return require "sqlite"(opts) + end, +}) --[[@as FrecencySqlite]] diff --git a/lua/frecency/tests/file_lock_spec.lua b/lua/frecency/tests/file_lock_spec.lua new file mode 100644 index 00000000..c6eba264 --- /dev/null +++ b/lua/frecency/tests/file_lock_spec.lua @@ -0,0 +1,135 @@ +local FileLock = require "frecency.file_lock" +local util = require "frecency.tests.util" +local async = require "plenary.async" --[[@as PlenaryAsync]] +require("plenary.async").tests.add_to_env() + +local function with_dir(f) + local dir, close = util.make_tree {} + local filename = (dir / "file_lock_test").filename + f(filename) + close() +end + +a.describe("file_lock", function() + a.describe("get()", function() + a.describe("when no lock file", function() + with_dir(function(filename) + local fl = FileLock.new(filename, { retry = 1, interval = 10 }) + a.it("gets successfully", function() + assert.is.Nil(fl:get()) + end) + end) + end) + + a.describe("when with a lock file", function() + with_dir(function(filename) + local fl = FileLock.new(filename, { retry = 1, interval = 10 }) + a.it("fails to get", function() + assert.is.Nil(async.uv.fs_open(fl.filename, "wx", tonumber("600", 8))) + assert.are.same("failed to get lock", fl:get()) + end) + end) + end) + + a.describe("when getting twice", function() + with_dir(function(filename) + local fl = FileLock.new(filename, { retry = 1, interval = 10 }) + a.it("fails to get", function() + assert.is.Nil(fl:get()) + assert.are.same("failed to get lock", fl:get()) + end) + end) + end) + end) + + a.describe("release()", function() + a.describe("when no lock file", function() + with_dir(function(filename) + local fl = FileLock.new(filename, { retry = 1, interval = 10 }) + a.it("fails to release", function() + assert.are.same("lock not found", fl:release()) + end) + end) + end) + + a.describe("when with a lock file", function() + with_dir(function(filename) + local fl = FileLock.new(filename, { retry = 1, interval = 10 }) + a.it("releases successfully", function() + assert.is.Nil(fl:get()) + assert.is.Nil(fl:release()) + end) + end) + end) + + a.describe("when releasing twice", function() + with_dir(function(filename) + local fl = FileLock.new(filename, { retry = 1, interval = 10 }) + a.it("fails to release", function() + assert.is.Nil(fl:get()) + assert.is.Nil(fl:release()) + assert.are.same("lock not found", fl:release()) + end) + end) + end) + end) + + a.describe("with()", function() + a.describe("when get() fails", function() + with_dir(function(filename) + local fl = FileLock.new(filename, { retry = 1, interval = 10 }) + a.it("fails with a valid err", function() + assert.is.Nil(fl:get()) + assert.are.same( + "failed to get lock", + fl:with(function() + return nil + end) + ) + end) + end) + end) + + a.describe("when release() fails", function() + with_dir(function(filename) + local fl = FileLock.new(filename, { retry = 1, interval = 10 }) + a.it("fails with a valid err", function() + assert.are.same( + "lock not found", + fl:with(function() + assert.is.Nil(async.uv.fs_unlink(fl.filename)) + return nil + end) + ) + end) + end) + end) + + a.describe("when f() fails", function() + with_dir(function(filename) + local fl = FileLock.new(filename, { retry = 1, interval = 10 }) + a.it("fails with a valid err", function() + assert.has.match( + ": error in hoge function$", + fl:with(function() + error "error in hoge function" + end) + ) + end) + end) + end) + + a.describe("when no errors", function() + with_dir(function(filename) + local fl = FileLock.new(filename, { retry = 1, interval = 10 }) + a.it("run successfully and returns valid results", function() + local err, result = fl:with(function() + return "hogehogeo" + end) + assert.is.Nil(err) + assert.are.same("hogehogeo", result) + end) + end) + end) + end) +end) diff --git a/lua/frecency/tests/frecency_spec.lua b/lua/frecency/tests/frecency_spec.lua index 769e0a23..cf8362c0 100644 --- a/lua/frecency/tests/frecency_spec.lua +++ b/lua/frecency/tests/frecency_spec.lua @@ -2,15 +2,18 @@ local Frecency = require "frecency.frecency" local Picker = require "frecency.picker" local util = require "frecency.tests.util" -local Path = require "plenary.path" local log = require "plenary.log" +local Path = require "plenary.path" + +local use_sqlite ---@param files string[] ---@param callback fun(frecency: Frecency, dir: PlenaryPath): nil ---@return nil local function with_files(files, callback) local dir, close = util.make_tree(files) - local frecency = Frecency.new { db_root = dir.filename } + log.debug { db_root = dir.filename } + local frecency = Frecency.new { db_root = dir.filename, use_sqlite = use_sqlite } frecency.picker = Picker.new( frecency.database, frecency.finder, @@ -41,7 +44,6 @@ local function make_register(frecency, dir) end end ----comment ---@param frecency Frecency ---@param dir PlenaryPath ---@param callback fun(register: fun(file: string, datetime: string?): nil): nil @@ -86,292 +88,304 @@ local function with_fake_vim_ui_select(choice, callback) end describe("frecency", function() - describe("register", function() - describe("when opening files", function() - with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) - local register = make_register(frecency, dir) - register("hoge1.txt", "2023-07-29T00:00:00+09:00") - register("hoge2.txt", "2023-07-29T01:00:00+09:00") - - it("has valid records in DB", function() - local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") - assert.are.same({ - { count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, - { count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, - }, results) - end) - end) - end) + local function test(db) + describe(db, function() + describe("register", function() + describe("when opening files", function() + with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) + local register = make_register(frecency, dir) + register("hoge1.txt", "2023-07-29T00:00:00+09:00") + register("hoge2.txt", "2023-07-29T01:00:00+09:00") - describe("when opening again", function() - with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) - local register = make_register(frecency, dir) - register("hoge1.txt", "2023-07-29T00:00:00+09:00") - register("hoge2.txt", "2023-07-29T01:00:00+09:00") - register("hoge1.txt", "2023-07-29T02:00:00+09:00", true) - - it("increases the score", function() - local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00") - assert.are.same({ - { count = 2, id = 1, path = filepath(dir, "hoge1.txt"), score = 40 }, - { count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, - }, results) + it("has valid records in DB", function() + local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") + assert.are.same({ + { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, + }, results) + end) + end) end) - end) - end) - describe("when opening again but the same instance", function() - with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) - local register = make_register(frecency, dir) - register("hoge1.txt", "2023-07-29T00:00:00+09:00") - register("hoge2.txt", "2023-07-29T01:00:00+09:00") - register("hoge1.txt", "2023-07-29T02:00:00+09:00") - - it("does not increase the score", function() - local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00") - assert.are.same({ - { count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, - { count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, - }, results) - end) - end) - end) + describe("when opening again", function() + with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) + local register = make_register(frecency, dir) + register("hoge1.txt", "2023-07-29T00:00:00+09:00") + register("hoge2.txt", "2023-07-29T01:00:00+09:00") + register("hoge1.txt", "2023-07-29T02:00:00+09:00", true) - describe("when opening more than 10 times", function() - with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) - local register = make_register(frecency, dir) - register("hoge1.txt", "2023-07-29T00:00:00+09:00") - register("hoge1.txt", "2023-07-29T00:01:00+09:00", true) - - register("hoge2.txt", "2023-07-29T00:00:00+09:00") - register("hoge2.txt", "2023-07-29T00:01:00+09:00", true) - register("hoge2.txt", "2023-07-29T00:02:00+09:00", true) - register("hoge2.txt", "2023-07-29T00:03:00+09:00", true) - register("hoge2.txt", "2023-07-29T00:04:00+09:00", true) - register("hoge2.txt", "2023-07-29T00:05:00+09:00", true) - register("hoge2.txt", "2023-07-29T00:06:00+09:00", true) - register("hoge2.txt", "2023-07-29T00:07:00+09:00", true) - register("hoge2.txt", "2023-07-29T00:08:00+09:00", true) - register("hoge2.txt", "2023-07-29T00:09:00+09:00", true) - register("hoge2.txt", "2023-07-29T00:10:00+09:00", true) - register("hoge2.txt", "2023-07-29T00:11:00+09:00", true) - - it("calculates score from the recent 10 times", function() - local results = frecency.picker:fetch_results(nil, "2023-07-29T00:12:00+09:00") - assert.are.same({ - { count = 12, id = 2, path = filepath(dir, "hoge2.txt"), score = 12 * (10 * 100) / 10 }, - { count = 2, id = 1, path = filepath(dir, "hoge1.txt"), score = 2 * (2 * 100) / 10 }, - }, results) - end) - end) - end) - end) - - describe("benchmark", function() - describe("after registered over >5000 files", function() - with_files({}, function(frecency, dir) - with_fake_register(frecency, dir, function(register) - local file_count = 6000 - if not os.getenv "CI" then - log.info "It works not on CI. Files is decreaed into 10 count." - file_count = 10 - end - local expected = {} - for i = 1, file_count do - local file = ("hoge%08d.txt"):format(i) - table.insert(expected, { count = 1, id = i, path = filepath(dir, file), score = 10 }) - register(file, "2023-07-29T00:00:00+09:00") - end - local start = os.clock() - local results = frecency.picker:fetch_results(nil, "2023-07-29T00:01:00+09:00") - table.sort(results, function(a, b) - return a.path < b.path + it("increases the score", function() + local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00") + assert.are.same({ + { count = 2, path = filepath(dir, "hoge1.txt"), score = 40 }, + { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 }, + }, results) + end) end) - local elapsed = os.clock() - start - log.info(("it takes %f seconds in fetching all results"):format(elapsed)) + end) - it("returns appropriate latency (<1.0 second)", function() - assert.are.is_true(elapsed < 1.0) - end) + describe("when opening again but the same instance", function() + with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) + local register = make_register(frecency, dir) + register("hoge1.txt", "2023-07-29T00:00:00+09:00") + register("hoge2.txt", "2023-07-29T01:00:00+09:00") + register("hoge1.txt", "2023-07-29T02:00:00+09:00") - it("returns valid response", function() - assert.are.same(expected, results) + it("does not increase the score", function() + local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00") + assert.are.same({ + { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, + }, results) + end) end) end) - end) - end) - end) - - describe("validate_database", function() - describe("when no files are unlinked", function() - with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) - local register = make_register(frecency, dir) - register("hoge1.txt", "2023-07-29T00:00:00+09:00") - register("hoge2.txt", "2023-07-29T00:01:00+09:00") - - it("removes no entries", function() - local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") - assert.are.same({ - { count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, - { count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, - }, results) - end) - end) - end) - describe("when with not force", function() - describe("when files are unlinked but it is less than threshold", function() - with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir) - local register = make_register(frecency, dir) - register("hoge1.txt", "2023-07-29T00:00:00+09:00") - register("hoge2.txt", "2023-07-29T00:01:00+09:00") - register("hoge3.txt", "2023-07-29T00:02:00+09:00") - register("hoge4.txt", "2023-07-29T00:03:00+09:00") - register("hoge5.txt", "2023-07-29T00:04:00+09:00") - frecency.config.db_validate_threshold = 3 - dir:joinpath("hoge1.txt"):rm() - dir:joinpath("hoge2.txt"):rm() - frecency:validate_database() - - it("removes no entries", function() - local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") - table.sort(results, function(a, b) - return a.path < b.path + describe("when opening more than 10 times", function() + with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) + local register = make_register(frecency, dir) + register("hoge1.txt", "2023-07-29T00:00:00+09:00") + register("hoge1.txt", "2023-07-29T00:01:00+09:00", true) + + register("hoge2.txt", "2023-07-29T00:00:00+09:00") + register("hoge2.txt", "2023-07-29T00:01:00+09:00", true) + register("hoge2.txt", "2023-07-29T00:02:00+09:00", true) + register("hoge2.txt", "2023-07-29T00:03:00+09:00", true) + register("hoge2.txt", "2023-07-29T00:04:00+09:00", true) + register("hoge2.txt", "2023-07-29T00:05:00+09:00", true) + register("hoge2.txt", "2023-07-29T00:06:00+09:00", true) + register("hoge2.txt", "2023-07-29T00:07:00+09:00", true) + register("hoge2.txt", "2023-07-29T00:08:00+09:00", true) + register("hoge2.txt", "2023-07-29T00:09:00+09:00", true) + register("hoge2.txt", "2023-07-29T00:10:00+09:00", true) + register("hoge2.txt", "2023-07-29T00:11:00+09:00", true) + + it("calculates score from the recent 10 times", function() + local results = frecency.picker:fetch_results(nil, "2023-07-29T00:12:00+09:00") + assert.are.same({ + { count = 12, path = filepath(dir, "hoge2.txt"), score = 12 * (10 * 100) / 10 }, + { count = 2, path = filepath(dir, "hoge1.txt"), score = 2 * (2 * 100) / 10 }, + }, results) end) - assert.are.same({ - { count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, - { count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, - { count = 1, id = 3, path = filepath(dir, "hoge3.txt"), score = 10 }, - { count = 1, id = 4, path = filepath(dir, "hoge4.txt"), score = 10 }, - { count = 1, id = 5, path = filepath(dir, "hoge5.txt"), score = 10 }, - }, results) end) end) end) - describe("when files are unlinked and it is more than threshold", function() - describe('when the user response "yes"', function() - with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir) - local register = make_register(frecency, dir) - register("hoge1.txt", "2023-07-29T00:00:00+09:00") - register("hoge2.txt", "2023-07-29T00:01:00+09:00") - register("hoge3.txt", "2023-07-29T00:02:00+09:00") - register("hoge4.txt", "2023-07-29T00:03:00+09:00") - register("hoge5.txt", "2023-07-29T00:04:00+09:00") - frecency.config.db_validate_threshold = 3 - dir:joinpath("hoge1.txt"):rm() - dir:joinpath("hoge2.txt"):rm() - dir:joinpath("hoge3.txt"):rm() - - with_fake_vim_ui_select("y", function(called) - frecency:validate_database() + describe("benchmark", function() + describe("after registered over >5000 files", function() + with_files({}, function(frecency, dir) + with_fake_register(frecency, dir, function(register) + -- TODO: 6000 records is too many to use with native? + -- local file_count = 6000 + local file_count = 600 + if not os.getenv "CI" then + log.info "It works not on CI. Files is decreased into 10 count." + file_count = 10 + end + local expected = {} + log.info(("making %d files and register them"):format(file_count)) + for i = 1, file_count do + local file = ("hoge%08d.txt"):format(i) + table.insert(expected, { count = 1, path = filepath(dir, file), score = 10 }) + register(file, "2023-07-29T00:00:00+09:00") + end + local start = os.clock() + local results = frecency.picker:fetch_results(nil, "2023-07-29T00:01:00+09:00") + table.sort(results, function(a, b) + return a.path < b.path + end) + local elapsed = os.clock() - start + log.info(("it takes %f seconds in fetching all results"):format(elapsed)) - it("called vim.ui.select()", function() - assert.are.same(1, called()) + it("returns appropriate latency (<1.0 second)", function() + assert.are.is_true(elapsed < 1.0) end) - end) - it("removes entries", function() - local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") - table.sort(results, function(a, b) - return a.path < b.path + it("returns valid response", function() + assert.are.same(expected, results) end) - assert.are.same({ - { count = 1, id = 4, path = filepath(dir, "hoge4.txt"), score = 10 }, - { count = 1, id = 5, path = filepath(dir, "hoge5.txt"), score = 10 }, - }, results) end) end) end) + end) - describe('when the user response "no"', function() - with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir) + describe("validate_database", function() + describe("when no files are unlinked", function() + with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) local register = make_register(frecency, dir) register("hoge1.txt", "2023-07-29T00:00:00+09:00") register("hoge2.txt", "2023-07-29T00:01:00+09:00") - register("hoge3.txt", "2023-07-29T00:02:00+09:00") - register("hoge4.txt", "2023-07-29T00:03:00+09:00") - register("hoge5.txt", "2023-07-29T00:04:00+09:00") - frecency.config.db_validate_threshold = 3 - dir:joinpath("hoge1.txt"):rm() - dir:joinpath("hoge2.txt"):rm() - dir:joinpath("hoge3.txt"):rm() - - with_fake_vim_ui_select("n", function(called) - frecency:validate_database() - - it("called vim.ui.select()", function() - assert.are.same(1, called()) - end) - end) it("removes no entries", function() local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") - table.sort(results, function(a, b) - return a.path < b.path - end) assert.are.same({ - { count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, - { count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, - { count = 1, id = 3, path = filepath(dir, "hoge3.txt"), score = 10 }, - { count = 1, id = 4, path = filepath(dir, "hoge4.txt"), score = 10 }, - { count = 1, id = 5, path = filepath(dir, "hoge5.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, }, results) end) end) end) - end) - end) - - describe("when with force", function() - describe("when db_safe_mode is true", function() - with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) - local register = make_register(frecency, dir) - register("hoge1.txt", "2023-07-29T00:00:00+09:00") - register("hoge2.txt", "2023-07-29T00:01:00+09:00") - dir:joinpath("hoge1.txt"):rm() - with_fake_vim_ui_select("y", function(called) - frecency:validate_database(true) + describe("when with not force", function() + describe("when files are unlinked but it is less than threshold", function() + with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir) + local register = make_register(frecency, dir) + register("hoge1.txt", "2023-07-29T00:00:00+09:00") + register("hoge2.txt", "2023-07-29T00:01:00+09:00") + register("hoge3.txt", "2023-07-29T00:02:00+09:00") + register("hoge4.txt", "2023-07-29T00:03:00+09:00") + register("hoge5.txt", "2023-07-29T00:04:00+09:00") + frecency.config.db_validate_threshold = 3 + dir:joinpath("hoge1.txt"):rm() + dir:joinpath("hoge2.txt"):rm() + frecency:validate_database() - it("called vim.ui.select()", function() - assert.are.same(1, called()) + it("removes no entries", function() + local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") + table.sort(results, function(a, b) + return a.path < b.path + end) + assert.are.same({ + { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge3.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge4.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge5.txt"), score = 10 }, + }, results) + end) end) end) - it("needs confirmation for removing entries", function() - local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") - assert.are.same({ - { count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, - }, results) + describe("when files are unlinked and it is more than threshold", function() + describe('when the user response "yes"', function() + with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir) + local register = make_register(frecency, dir) + register("hoge1.txt", "2023-07-29T00:00:00+09:00") + register("hoge2.txt", "2023-07-29T00:01:00+09:00") + register("hoge3.txt", "2023-07-29T00:02:00+09:00") + register("hoge4.txt", "2023-07-29T00:03:00+09:00") + register("hoge5.txt", "2023-07-29T00:04:00+09:00") + frecency.config.db_validate_threshold = 3 + dir:joinpath("hoge1.txt"):rm() + dir:joinpath("hoge2.txt"):rm() + dir:joinpath("hoge3.txt"):rm() + + with_fake_vim_ui_select("y", function(called) + frecency:validate_database() + + it("called vim.ui.select()", function() + assert.are.same(1, called()) + end) + end) + + it("removes entries", function() + local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") + table.sort(results, function(a, b) + return a.path < b.path + end) + assert.are.same({ + { count = 1, path = filepath(dir, "hoge4.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge5.txt"), score = 10 }, + }, results) + end) + end) + end) + + describe('when the user response "no"', function() + with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir) + local register = make_register(frecency, dir) + register("hoge1.txt", "2023-07-29T00:00:00+09:00") + register("hoge2.txt", "2023-07-29T00:01:00+09:00") + register("hoge3.txt", "2023-07-29T00:02:00+09:00") + register("hoge4.txt", "2023-07-29T00:03:00+09:00") + register("hoge5.txt", "2023-07-29T00:04:00+09:00") + frecency.config.db_validate_threshold = 3 + dir:joinpath("hoge1.txt"):rm() + dir:joinpath("hoge2.txt"):rm() + dir:joinpath("hoge3.txt"):rm() + + with_fake_vim_ui_select("n", function(called) + frecency:validate_database() + + it("called vim.ui.select()", function() + assert.are.same(1, called()) + end) + end) + + it("removes no entries", function() + local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") + table.sort(results, function(a, b) + return a.path < b.path + end) + assert.are.same({ + { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge3.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge4.txt"), score = 10 }, + { count = 1, path = filepath(dir, "hoge5.txt"), score = 10 }, + }, results) + end) + end) + end) end) end) - end) - describe("when db_safe_mode is false", function() - with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) - local register = make_register(frecency, dir) - register("hoge1.txt", "2023-07-29T00:00:00+09:00") - register("hoge2.txt", "2023-07-29T00:01:00+09:00") - dir:joinpath("hoge1.txt"):rm() + describe("when with force", function() + describe("when db_safe_mode is true", function() + with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) + local register = make_register(frecency, dir) + register("hoge1.txt", "2023-07-29T00:00:00+09:00") + register("hoge2.txt", "2023-07-29T00:01:00+09:00") + dir:joinpath("hoge1.txt"):rm() - with_fake_vim_ui_select("y", function(called) - frecency.config.db_safe_mode = false - frecency:validate_database(true) + with_fake_vim_ui_select("y", function(called) + frecency:validate_database(true) - it("did not call vim.ui.select()", function() - assert.are.same(0, called()) + it("called vim.ui.select()", function() + assert.are.same(1, called()) + end) + end) + + it("needs confirmation for removing entries", function() + local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") + assert.are.same({ + { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 }, + }, results) + end) end) end) - it("needs no confirmation for removing entries", function() - local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") - assert.are.same({ - { count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, - }, results) + describe("when db_safe_mode is false", function() + with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) + local register = make_register(frecency, dir) + register("hoge1.txt", "2023-07-29T00:00:00+09:00") + register("hoge2.txt", "2023-07-29T00:01:00+09:00") + dir:joinpath("hoge1.txt"):rm() + + with_fake_vim_ui_select("y", function(called) + frecency.config.db_safe_mode = false + frecency:validate_database(true) + + it("did not call vim.ui.select()", function() + assert.are.same(0, called()) + end) + end) + + it("needs no confirmation for removing entries", function() + local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") + assert.are.same({ + { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 }, + }, results) + end) + end) end) end) end) end) - end) + end + + use_sqlite = true + test "sqlite" + use_sqlite = false + test "native" end) diff --git a/lua/frecency/tests/migrator_spec.lua b/lua/frecency/tests/migrator_spec.lua new file mode 100644 index 00000000..7f9b5837 --- /dev/null +++ b/lua/frecency/tests/migrator_spec.lua @@ -0,0 +1,129 @@ +---@diagnostic disable: undefined-field +local Migrator = require "frecency.migrator" +local FS = require "frecency.fs" +local Recency = require "frecency.recency" +local Sqlite = require "frecency.database.sqlite" +local Native = require "frecency.database.native" +local util = require "frecency.tests.util" +local wait = require "frecency.wait" + +---@param callback fun(migrator: FrecencyMigrator, sqlite: FrecencyDatabase): nil +---@return nil +local function with(callback) + local dir, close = util.tmpdir() + local recency = Recency.new { max_count = 2 } + local fs = FS.new { ignore_patterns = {} } + local migrator = Migrator.new(fs, recency, dir.filename) + local sqlite = Sqlite.new(fs, { root = dir.filename }) + callback(migrator, sqlite) + close() +end + +---@param source table +local function v1_table(source) + local records = {} + for path, record in pairs(source) do + local timestamps = {} + for _, timestamp in ipairs(record.timestamps) do + table.insert(timestamps, vim.fn.strptime("%FT%T%z", timestamp)) + end + records[path] = { count = record.count, timestamps = timestamps } + end + return { version = "v1", records = records } +end + +describe("migrator", function() + describe("to_v1", function() + describe("when with simple database", function() + with(function(migrator, sqlite) + for _, path in ipairs { "hoge1.txt", "hoge2.txt" } do + sqlite:update(path, migrator.recency.config.max_count, "2023-08-21T00:00:00") + end + migrator:to_v1() + local native = Native.new(migrator.fs, { root = migrator.root }) + + it("has converted into a valid table", function() + assert.are.same( + v1_table { + ["hoge1.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00+0000" } }, + ["hoge2.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00+0000" } }, + }, + native.table + ) + end) + end) + end) + + describe("when with more large database", function() + with(function(migrator, sqlite) + for i, path in ipairs { + "hoge1.txt", + "hoge1.txt", + "hoge1.txt", + "hoge1.txt", + "hoge2.txt", + "hoge2.txt", + "hoge2.txt", + "hoge3.txt", + "hoge3.txt", + "hoge4.txt", + } do + sqlite:update(path, migrator.recency.config.max_count, ("2023-08-21T00:%02d:00"):format(i)) + end + migrator:to_v1() + local native = Native.new(migrator.fs, { root = migrator.root }) + + it("has converted into a valid table", function() + assert.are.same( + v1_table { + ["hoge1.txt"] = { count = 4, timestamps = { "2023-08-21T00:03:00+0000", "2023-08-21T00:04:00+0000" } }, + ["hoge2.txt"] = { count = 3, timestamps = { "2023-08-21T00:06:00+0000", "2023-08-21T00:07:00+0000" } }, + ["hoge3.txt"] = { count = 2, timestamps = { "2023-08-21T00:08:00+0000", "2023-08-21T00:09:00+0000" } }, + ["hoge4.txt"] = { count = 1, timestamps = { "2023-08-21T00:10:00+0000" } }, + }, + native.table + ) + end) + end) + end) + end) + + describe("to_sqlite", function() + with(function(migrator, sqlite) + local native = Native.new(migrator.fs, { root = migrator.root }) + native.table = v1_table { + ["hoge1.txt"] = { count = 4, timestamps = { "2023-08-21T00:03:00+0000", "2023-08-21T00:04:00+0000" } }, + ["hoge2.txt"] = { count = 3, timestamps = { "2023-08-21T00:06:00+0000", "2023-08-21T00:07:00+0000" } }, + ["hoge3.txt"] = { count = 2, timestamps = { "2023-08-21T00:08:00+0000", "2023-08-21T00:09:00+0000" } }, + ["hoge4.txt"] = { count = 1, timestamps = { "2023-08-21T00:10:00+0000" } }, + } + wait(function() + native:save() + end) + migrator:to_sqlite() + sqlite.sqlite.db:open() + local records = sqlite.sqlite.db:eval [[ + select + f.path, + f.count, + datetime(strftime('%s', t.timestamp), 'unixepoch') datetime + from timestamps t + join files f + on f.id = t.file_id + order by path, datetime + ]] + + it("has converted into a valid DB", function() + assert.are.same({ + { path = "hoge1.txt", count = 4, datetime = "2023-08-21 00:03:00" }, + { path = "hoge1.txt", count = 4, datetime = "2023-08-21 00:04:00" }, + { path = "hoge2.txt", count = 3, datetime = "2023-08-21 00:06:00" }, + { path = "hoge2.txt", count = 3, datetime = "2023-08-21 00:07:00" }, + { path = "hoge3.txt", count = 2, datetime = "2023-08-21 00:08:00" }, + { path = "hoge3.txt", count = 2, datetime = "2023-08-21 00:09:00" }, + { path = "hoge4.txt", count = 1, datetime = "2023-08-21 00:10:00" }, + }, records) + end) + end) + end) +end) diff --git a/lua/frecency/tests/util.lua b/lua/frecency/tests/util.lua index 8227915d..b776d3f3 100644 --- a/lua/frecency/tests/util.lua +++ b/lua/frecency/tests/util.lua @@ -1,17 +1,25 @@ local uv = vim.uv or vim.loop local Path = require "plenary.path" +---@return PlenaryPath +---@return fun(): nil close swwp all entries +local function tmpdir() + local dir = Path:new(Path:new(assert(uv.fs_mkdtemp "tests_XXXXXX")):absolute()) + return dir, function() + dir:rm { recursive = true } + end +end + ---@param entries string[] ----@return PlenaryPath the top dir of tree ----@return fun(): nil sweep all entries +---@return PlenaryPath dir the top dir of tree +---@return fun(): nil close sweep all entries local function make_tree(entries) - local dir = Path:new(Path.new(assert(uv.fs_mkdtemp "tests_XXXXXX")):absolute()) + local dir, close = tmpdir() for _, entry in ipairs(entries) do + ---@diagnostic disable-next-line: undefined-field dir:joinpath(entry):touch { parents = true } end - return dir, function() - dir:rm { recursive = true } - end + return dir, close end -return { make_tree = make_tree } +return { make_tree = make_tree, tmpdir = tmpdir } diff --git a/lua/frecency/types.lua b/lua/frecency/types.lua index e7fcd72e..6594dc17 100644 --- a/lua/frecency/types.lua +++ b/lua/frecency/types.lua @@ -1,4 +1,5 @@ ----@diagnostic disable: unused-local +---@diagnostic disable: unused-local, missing-return + -- NOTE: types below are borrowed from sqlite.lua ---@class sqlite_db @Main sqlite.lua object. @@ -51,6 +52,7 @@ ---@field filename string ---@field joinpath fun(self: PlenaryPath, ...): PlenaryPath ---@field make_relative fun(self: PlenaryPath, cwd: string): string +---@field parent PlenaryPath ---@field path { sep: string } ---@field rm fun(self: PlenaryPath, opts: { recursive: boolean }?): nil @@ -69,6 +71,8 @@ ---@class PlenaryAsync ---@field control PlenaryAsyncControl ---@field util PlenaryAsyncUtil +---@field uv PlenaryAsyncUv +---@field void fun(f: fun(): nil): fun(): nil local PlenaryAsync = {} ---@async @@ -96,6 +100,50 @@ function PlenaryAsyncControlChannelRx.recv() end ---@class PlenaryAsyncUtil local PlenaryAsyncUtil = {} +---@class PlenaryAsyncUv +local PlenaryAsyncUv = {} + +---@async +---@param path string +---@return string? err +---@return { mtime: integer, size: integer, type: "file"|"directory" } +function PlenaryAsyncUv.fs_stat(path) end + +---@async +---@param path string +---@param flags string|integer +---@param mode integer +---@return string? err +---@return integer fd +function PlenaryAsyncUv.fs_open(path, flags, mode) end + +---@async +---@param fd integer +---@param size integer +---@param offset integer? +---@return string? err +---@return string data +function PlenaryAsyncUv.fs_read(fd, size, offset) end + +---@async +---@param fd integer +---@param data string +---@param offset integer? +---@return string? err +---@return integer bytes +function PlenaryAsyncUv.fs_write(fd, data, offset) end + +---@async +---@param path string +---@return string? err +---@return boolean? success +function PlenaryAsyncUv.fs_unlink(path) end + +---@async +---@param fd integer +---@return string? err +function PlenaryAsyncUv.fs_close(fd) end + ---@async ---@param ms integer ---@return nil diff --git a/lua/frecency/wait.lua b/lua/frecency/wait.lua new file mode 100644 index 00000000..a27efd9b --- /dev/null +++ b/lua/frecency/wait.lua @@ -0,0 +1,54 @@ +local async = require "plenary.async" + +---@class FrecencyWait +---@field config FrecencyWaitConfig +local Wait = {} + +---@class FrecencyWaitConfig +---@field time integer default: 5000 +---@field interval integer default: 200 + +---@alias FrecencyWaitCallback fun(): nil + +---@param f FrecencyWaitCallback +---@param opts FrecencyWaitConfig? +Wait.new = function(f, opts) + return setmetatable( + { f = f, config = vim.tbl_extend("force", { time = 5000, interval = 200 }, opts or {}) }, + { __index = Wait } + ) +end + +---@async +---@private +Wait.f = function() + error "implement me" +end + +---@return boolean ok +---@return nil|-1|-2 status +function Wait:run() + local done = false + async.void(function() + self.f() + done = true + end)() + return vim.wait(self.config.time, function() + return done + end, self.config.interval) +end + +---@param f FrecencyWaitCallback +---@param opts FrecencyWaitConfig? +---@return nil +return function(f, opts) + local wait = Wait.new(f, opts) + local ok, status = wait:run() + if ok then + return + elseif status == -1 then + error "callback never returnes during the time" + elseif status == -2 then + error "callback is interrupted during the time" + end +end diff --git a/lua/telescope/_extensions/frecency.lua b/lua/telescope/_extensions/frecency.lua index f7eee955..481b6d86 100644 --- a/lua/telescope/_extensions/frecency.lua +++ b/lua/telescope/_extensions/frecency.lua @@ -1,12 +1,13 @@ local frecency = require "frecency" +local sqlite = require "frecency.sqlite" return require("telescope").register_extension { setup = frecency.setup, health = function() - if vim.F.npcall(require, "sqlite") then + if sqlite.can_use then vim.health.ok "sqlite.lua installed." else - vim.health.error "sqlite.lua is required for telescope-frecency.nvim to work." + vim.health.info "sqlite.lua is required when use_sqlite = true" end if vim.F.npcall(require, "nvim-web-devicons") then vim.health.ok "nvim-web-devicons installed."