feat(config): import option for splitting rocks toml configs (#565)

* feat: Add support for config imports

* Add tests for import feature

* Use newer version of toml-edit and add more tests

* Update flake.lock

* Update README.md

* Update lua/rocks/operations/helpers/multi_mut_rocks_toml_wrapper.lua

---------

Co-authored-by: Marc Jakobi <marc@jakobi.dev>
Co-authored-by: Marc Jakobi <mrcjkb89@outlook.com>

.github/workflows/luarocks.yml

@@ -32,7 +32,7 @@ jobs:
           test_interpreters: ""
           dependencies: |
             luarocks >= 3.11.1, < 4.0.0
-            toml-edit >= 0.3.6
+            toml-edit >= 0.5.0
             fidget.nvim >= 1.1.0
             fzy
             nvim-nio

README.md

@@ -454,6 +454,28 @@ You can also pin/unpin installed plugins with:
 :Rocks [pin|unpin] {rock}
 ```
 
+### Importing rocks toml files
+
+You can break up your rocks configuration into different modules that
+can then be imported into your main configuration. This can be useful
+for modularity or simply for the purpose of supporting local
+configuration files that you can keep outside of version control.
+
+For example:
+
+```toml
+import = [
+  "rocks-local.toml", # Paths are relative to the rocks.toml file directory by default
+  "~/my-rocks.toml", # Path expansion is supported through vim.fn.expand
+  "/home/user/my-rocks.toml", # Absolute paths are supported
+]
+
+```
+
+> [!NOTE]
+>
+> - Imported config will have higher priority
+
 ## :calendar: User events
 
 For `:h User` events that rocks.nvim will trigger, see `:h rocks.user-event`.

doc/rocks.txt

@@ -330,6 +330,7 @@ RocksToml                                                            *RocksToml*
         {plugins?}      (table<rock_name,RockSpec>)  The `[plugins]` entries
         {servers?}      (string[])
         {dev_servers?}  (string[])
+        {import?}       (string[])
         {string}        (unknown)                    Fields that can be added by external modules
 
 

lua/rocks/api/init.lua

@@ -117,6 +117,7 @@ end
 ---@field plugins? table<rock_name, RockSpec> The `[plugins]` entries
 ---@field servers? string[]
 ---@field dev_servers? string[]
+---@field import? string[]
 ---@field [string] unknown Fields that can be added by external modules
 
 ---Returns a table with the parsed rocks.toml file.

lua/rocks/config/internal.lua

@@ -38,6 +38,8 @@ end
 
 local default_luarocks_binary = get_default_luarocks_binary(default_rocks_path)
 
+local notified_recursive_imports = {}
+
 --- rocks.nvim default configuration
 ---@class RocksConfig
 local default_config = {
@@ -70,26 +72,74 @@ local default_config = {
         ---@type string[]
         unrecognized_configs = {},
     },
+    ---@type fun(parse_func: (fun(file_str: string, file_path: string): table), process_func: fun(config: table, file_path))
+    read_rocks_toml = function(parse_func, process_func)
+        local visited = {}
+
+        local function parse(file_path, default)
+            -- Don't allow recursive includes
+            if visited[file_path] then
+                if not notified_recursive_imports[file_path] then
+                    vim.defer_fn(function()
+                        vim.notify("Recursive import detected: " .. file_path, vim.log.levels.WARN)
+                    end, 1000)
+                    notified_recursive_imports[file_path] = true
+                end
+                return nil
+            end
+            visited[file_path] = true
+
+            -- Read config
+            local file_str = fs.read_or_create(file_path, default)
+            -- Parse
+            local rocks_toml = parse_func(file_str, file_path)
+            -- Follow import paths (giving preference to imported config)
+            if rocks_toml.import then
+                -- NOTE: using a while loop as the imports may be a metatable
+                local i, import_path = 0, nil
+                while true do
+                    i = i + 1
+                    import_path = rocks_toml.import[i]
+                    if import_path == nil then
+                        break
+                    end
+                    parse(fs.get_absolute_path(vim.fs.dirname(config.config_path), import_path), "")
+                end
+            end
+            -- Process result
+            process_func(rocks_toml, file_path)
+        end
+        parse(config.config_path, constants.DEFAULT_CONFIG)
+    end,
     ---@type fun():RocksToml
     get_rocks_toml = function()
-        local config_file = fs.read_or_create(config.config_path, constants.DEFAULT_CONFIG)
-        local rocks_toml = require("toml_edit").parse_as_tbl(config_file)
-        for key, tbl in pairs(rocks_toml) do
-            if key == "rocks" or key == "plugins" then
-                for name, data in pairs(tbl) do
-                    if type(data) == "string" then
-                        ---@type RockSpec
-                        rocks_toml[key][name] = {
-                            name = name,
-                            version = data,
-                        }
-                    else
-                        rocks_toml[key][name].name = name
+        local rocks_toml_merged = {}
+        config.read_rocks_toml(function(file_str, _)
+            -- Parse
+            return require("toml_edit").parse_as_tbl(file_str)
+        end, function(rocks_toml, _)
+            -- Setup rockspec for rocks/plugins
+            for key, tbl in pairs(rocks_toml) do
+                if key == "rocks" or key == "plugins" then
+                    for name, data in pairs(tbl) do
+                        if type(data) == "string" then
+                            ---@type RockSpec
+                            rocks_toml[key][name] = {
+                                name = name,
+                                version = data,
+                            }
+                        else
+                            rocks_toml[key][name].name = name
+                        end
                     end
                 end
             end
-        end
-        return rocks_toml
+            -- Merge into configuration, in the order of preference returned by the read function
+            rocks_toml_merged = vim.tbl_deep_extend("keep", rocks_toml_merged, rocks_toml)
+        end)
+        rocks_toml_merged.import = nil -- Remove import field since we merged
+
+        return rocks_toml_merged
     end,
     ---@return server_url[]
     get_servers = function()

lua/rocks/fs.lua

@@ -36,6 +36,35 @@ function fs.file_exists(location)
     return false
 end
 
+--- Expand environment variables and tilde in the path string
+---@param path_str string
+---@return string
+function fs.expand_path(path_str)
+    -- Expand environment variables
+    local path = path_str:gsub("%$([%w_]+)", function(var)
+        return os.getenv(var) or ""
+    end)
+    -- Expand tilde to home directory
+    local home = os.getenv("HOME")
+    if home then
+        path = path:gsub("^~", home)
+    end
+    return path
+end
+
+--- Expand path string and get the absolute path if it a relative path string
+---@param base_path string base directory path to use if path_str is relative
+---@param path_str string the path string to expand
+---@return string
+function fs.get_absolute_path(base_path, path_str)
+    local path = fs.expand_path(path_str)
+    -- If path is not an absolute path, set it relative to the base
+    if path:sub(1, 1) ~= "/" then
+        path = vim.fs.joinpath(fs.expand_path(base_path), path)
+    end
+    return path
+end
+
 --- Write `contents` to a file asynchronously
 ---@param location string file path
 ---@param mode string mode to open the file for
@@ -66,7 +95,9 @@ function fs.write_file(location, mode, contents, callback)
         else
             local msg = ("Error opening %s for writing: %s"):format(location, err)
             log.error(msg)
-            vim.notify(msg, vim.log.levels.ERROR)
+            vim.schedule(function()
+                vim.notify(msg, vim.log.levels.ERROR)
+            end)
             if callback then
                 callback()
             end

lua/rocks/operations/add.lua

@@ -18,8 +18,6 @@ local add = {}
 
 local constants = require("rocks.constants")
 local log = require("rocks.log")
-local fs = require("rocks.fs")
-local config = require("rocks.config.internal")
 local cache = require("rocks.cache")
 local helpers = require("rocks.operations.helpers")
 local handlers = require("rocks.operations.handlers")
@@ -98,7 +96,7 @@ add.add = function(arg_list, callback, opts)
                     })
                 end
                 handler(report_progress, report_error, helpers.manage_rock_stub)
-                fs.write_file_await(config.config_path, "w", tostring(user_rocks))
+                user_rocks:write()
                 nio.scheduler()
                 progress_handle:finish()
                 return
@@ -217,7 +215,7 @@ Use 'Rocks %s {rock_name}' or install rocks-git.nvim.
             else
                 user_rocks.plugins[rock_name] = installed_rock.version
             end
-            fs.write_file_await(config.config_path, "w", tostring(user_rocks))
+            user_rocks:write()
             cache.populate_all_rocks_state_caches()
             vim.schedule(function()
                 helpers.postInstall()

lua/rocks/operations/helpers.lua → lua/rocks/operations/helpers/init.lua

@@ -24,6 +24,7 @@ local state = require("rocks.state")
 local log = require("rocks.log")
 local cache = require("rocks.cache")
 local nio = require("nio")
+local multi_mut_rocks_toml_wrapper = require("rocks.operations.helpers.multi_mut_rocks_toml_wrapper")
 
 local helpers = {}
 
@@ -32,8 +33,18 @@ helpers.semaphore = nio.control.semaphore(1)
 ---Decode the user rocks from rocks.toml, creating a default config file if it does not exist
 ---@return MutRocksTomlRef
 function helpers.parse_rocks_toml()
-    local config_file = fs.read_or_create(config.config_path, constants.DEFAULT_CONFIG)
-    return require("toml_edit").parse(config_file)
+    local rocks_toml_configs = {}
+    config.read_rocks_toml(function(file_str, file_path)
+        -- Parse
+        return require("toml_edit").parse(file_str)
+    end, function(rocks_toml, file_path)
+        ---@type MutRocksTomlRefWithPath
+        local rocks_toml_config = { config = rocks_toml, path = file_path }
+        -- Append to config list in order of preference returned by the read function
+        table.insert(rocks_toml_configs, rocks_toml_config)
+    end)
+
+    return multi_mut_rocks_toml_wrapper.new(rocks_toml_configs) --[[@as MutRocksTomlRef]]
 end
 
 ---@param rocks_toml MutRocksTomlRef

lua/rocks/operations/helpers/multi_mut_rocks_toml_wrapper.lua

@@ -0,0 +1,75 @@
+local config = require("rocks.config.internal")
+local fs = require("rocks.fs")
+
+---@class MutRocksTomlRefWithPath
+---@field config MutRocksTomlRef Config metatable
+---@field path? string The path to the configuration
+
+---@class MultiMutRocksTomlWrapper
+---@field cache table<string, MultiMutRocksTomlWrapper> Cache for nested metatables
+---@field configs MutRocksTomlRefWithPath[] A list of rocks toml configs
+local MultiMutRocksTomlWrapper = {}
+MultiMutRocksTomlWrapper.__index = function(self, key)
+    -- Give preference to class methods/fields
+    if MultiMutRocksTomlWrapper[key] then
+        return MultiMutRocksTomlWrapper[key]
+    end
+    -- Find the key within the config tables
+    for _, tbl in ipairs(self.configs) do
+        if tbl.config[key] ~= nil then
+            if type(tbl.config[key]) == "table" then
+                if not self.cache[key] then
+                    self.cache[key] = MultiMutRocksTomlWrapper.new(vim.iter(self.configs)
+                        :filter(function(v)
+                            return type(v.config[key]) == "table"
+                        end)
+                        :fold({}, function(acc, v)
+                            table.insert(acc, { config = v.config[key], path = v.path })
+                            return acc
+                        end))
+                end
+                return self.cache[key]
+            else
+                return tbl.config[key]
+            end
+        end
+    end
+    return nil
+end
+MultiMutRocksTomlWrapper.__newindex = function(self, key, value)
+    local insert_index = 1
+    for i, tbl in ipairs(self.configs) do
+        -- Insert into base config by default
+        if tbl.path == config.config_path then
+            insert_index = i
+        end
+        if tbl.config[key] ~= nil then
+            tbl.config[key] = value
+            return
+        end
+    end
+    -- If key not found in any table, add it to the first table
+    self.configs[insert_index].config[key] = value
+end
+
+--- Write to all rocks toml config files in an async context
+---@type async fun(self: MultiMutRocksTomlWrapper)
+function MultiMutRocksTomlWrapper:write()
+    for _, tbl in ipairs(self.configs) do
+        if tbl.path ~= nil then
+            fs.write_file_await(tbl.path, "w", tostring(tbl.config))
+        end
+    end
+end
+
+--- Function to create a new wrapper
+---@param configs MutRocksTomlRefWithPath[] A list of rocks toml configs
+---@return MultiMutRocksTomlWrapper
+function MultiMutRocksTomlWrapper.new(configs)
+    assert(#configs > 0, "Must provide at least one rocks toml config")
+    local self = { cache = {}, configs = configs }
+    setmetatable(self, MultiMutRocksTomlWrapper)
+    return self
+end
+
+return MultiMutRocksTomlWrapper

lua/rocks/operations/pin.lua

@@ -16,8 +16,6 @@
 
 local pin = {}
 
-local fs = require("rocks.fs")
-local config = require("rocks.config.internal")
 local helpers = require("rocks.operations.helpers")
 local nio = require("nio")
 
@@ -40,7 +38,7 @@ pin.pin = function(rock_name)
             end
             user_config[rocks_key][rock_name].pin = true
             local version = user_config[rocks_key][rock_name].version
-            fs.write_file_await(config.config_path, "w", tostring(user_config))
+            user_config:write()
             vim.schedule(function()
                 vim.notify(("%s pinned to version %s"):format(rock_name, version), vim.log.levels.INFO)
             end)