From 5a3a06e5df34f1b4fc1500622b10620c49c60da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3n=C3=A1n=20Carrigan?= Date: Fri, 9 Feb 2024 21:23:00 +0000 Subject: [PATCH] feat: file module --- README.md | 62 ++++++++++++++++++++++--- doc/nio.txt | 93 ++++++++++++++++++++++++++++++++----- lua/nio/file.lua | 59 ++++++++++++++++++++++++ lua/nio/init.lua | 2 + lua/nio/process.lua | 46 +++++++++++++++---- lua/nio/streams.lua | 76 ++++++++++++++++++++++++------- lua/nio/tasks.lua | 6 +-- scripts/gendocs.lua | 1 + tests/file_spec.lua | 101 +++++++++++++++++++++++++++++++++++++++++ tests/process_spec.lua | 36 +++++++++++++++ 10 files changed, 436 insertions(+), 46 deletions(-) create mode 100644 lua/nio/file.lua create mode 100644 tests/file_spec.lua diff --git a/README.md b/README.md index efa5980..2704c4c 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,19 @@ both common asynchronous primitives and asynchronous APIs for Neovim's core. - [Installation](#installation) - [Configuration](#configuration) - [Usage](#usage) + - [`nio.control`](#niocontrol): Primitives for flow control in async functions + - [`nio.lsp`](#niolsp): A fully typed and documented async LSP client library, generated from the LSP specification. + - [`nio.file`](#niofile): Open and operate on files asynchronously + - [`nio.process`](#nioprocess): Run and control subprocesses asynchronously + - [`nio.uv`](#niouv): Async versions of `vim.loop` functions + - [`nio.ui`](#nioui): Async versions of vim.ui functions + - [`nio.tests`](#niotests): Async versions of plenary.nvim's test functions + - [Third Party Integration](#third-party-integration) ## Motivation Work has been ongoing around async libraries in Neovim for years, with a lot of discussion around a [Neovim core -implementation](https://github.com/neovim/neovim/issues/19624). A lot of the motivation behind this library can be seen +implementation](https://github.com/neovim/neovim/issues/19624). Much of the motivation behind this library can be seen in that discussion. nvim-nio aims to provide a simple interface to Lua coroutines that doesn't feel like it gets in the way of your actual @@ -82,7 +90,9 @@ For simple use cases tasks won't be too important but they support features such nvim-nio comes with built-in modules to help with writing async code. See `:help nvim-nio` for extensive documentation. -`nio.control`: Primitives for flow control in async functions +### `nio.control` + +Primitives for flow control in async functions ```lua local event = nio.control.event() @@ -104,7 +114,9 @@ local listeners = { } ``` -`nio.lsp`: A fully typed and documented async LSP client library, generated from the LSP specification. +### `nio.lsp` + +A fully typed and documented async LSP client library, generated from the LSP specification. ```lua local client = nio.lsp.get_clients({ name = "lua_ls" })[1] @@ -120,7 +132,39 @@ for _, token in pairs(response.data) do end ``` -`nio.uv`: Async versions of `vim.loop` functions +### `nio.file` + +Open and operate on files asynchronously + +```lua +local file = nio.file.open("test.txt", "w+") + +file.write("Hello, World!\n") + +local content = file.read(nil, 0) +print(content) +``` + +### `nio.process` + +Run and control subprocesses asynchronously + +```lua +local first = nio.process.run({ + cmd = "printf", args = { "hello" } +}) + +local second = nio.process.run({ + cmd = "cat", stdin = first.stdout +}) + +local output = second.stdout.read() +print(output) +``` + +### `nio.uv` + +Async versions of `vim.loop` functions ```lua local file_path = "README.md" @@ -140,14 +184,18 @@ assert(not close_err, close_err) print(data) ``` -`nio.ui`: Async versions of vim.ui functions +### `nio.ui` + +Async versions of vim.ui functions ```lua local value = nio.ui.input({ prompt = "Enter something: " }) print(("You entered: %s"):format(value)) ``` -`nio.tests`: Async versions of plenary.nvim's test functions +### `nio.tests` + +Async versions of plenary.nvim's test functions ```lua nio.tests.it("notifies listeners", function() @@ -166,6 +214,8 @@ nio.tests.it("notifies listeners", function() end) ``` +### Third Party Integration + It is also easy to wrap callback style functions to make them asynchronous using `nio.wrap`, which allows easily integrating third-party APIs with nvim-nio. diff --git a/doc/nio.txt b/doc/nio.txt index 1505786..49797ac 100644 --- a/doc/nio.txt +++ b/doc/nio.txt @@ -351,6 +351,47 @@ Return~ `(nio.lsp.Client)` +============================================================================== +nio.file *nio.file* + + + *nio.file.File* +Inherits: `nio.streams.OSStreamReaderWriter` + +Fields~ +{read} `(async fun(n: integer?, offset: integer?):string?,string?)` Read data +from the stream, optionally up to n bytes otherwise until EOF is reached. +Returns the data read or error message if an error occurred. If offset is +provided, data will be read from that position in the file, otherwise the +current position will be used. + + *nio.file.open()* +`open`({path}, {flags}, {mode}) + +Open a file with the given flags and mode +>lua + local file = nio.file.open("test.txt", "w+") + + file.write("Hello, World!\n") + + local content = file.read(nil, 0) + file.close() + print(content) +< +Parameters~ +{path} `(string)` The path to the file +{flags} `(uv.aliases.fs_access_flags|integer?)` The flags to open the file +with, defaults to "r" +{mode} `(number?)` The mode to open the file with, defaults to 644 +Return~ +`(nio.file.File?)` File object +Return~ +`(string?)` Error message if an error occurred while opening + +See also ~ +|uv.fs_open| + + ============================================================================== nio.process *nio.process* @@ -374,6 +415,17 @@ stderr. `run`({opts}) Run a process asynchronously. +>lua + local process = nio.process.run({ + cmd = "printf", args = { "hello" } + }) + + local output = second.stdout.read() + print(output) +< + +Processes can be chained together, passing output of one process as input to +another. >lua local first = nio.process.run({ cmd = "printf", args = { "hello" } @@ -386,6 +438,23 @@ Run a process asynchronously. local output = second.stdout.read() print(output) < + +The stdio fields can also be file objects. +>lua + local path = nio.fn.tempname() + + local file = nio.file.open(path, "w+") + + local process = nio.process.run({ + cmd = "printf", + args = { "hello" }, + stdout = file, + }) + process.result() + + local output = file.read(nil, 0) + print(output) +< Parameters~ {opts} `(nio.process.RunOpts)` Return~ @@ -397,14 +466,12 @@ Return~ Fields~ {cmd} `(string)` Command to run {args?} `(string[])` Arguments to pass to the command -{stdin?} `(integer|nio.streams.OSStreamReader|uv.uv_pipe_t|uv_pipe_t)` Stream, -pipe or file descriptor to use as stdin. -{stdout?} `(integer|nio.streams.OSStreamWriter|uv.uv_pipe_t|uv_pipe_t)` -Stream, -pipe or file descriptor to use as stdout. -{stderr?} `(integer|nio.streams.OSStreamWriter|uv.uv_pipe_t|uv_pipe_t)` -Stream, -pipe or file descriptor to use as stderr. +{stdin?} `(integer|nio.streams.OSStream|uv_pipe_t)` Stream, pipe or file +to use as stdin. +{stdout?} `(integer|nio.streams.OSStream|uv_pipe_t)` Stream, pipe or file +to use as stdout. +{stderr?} `(integer|nio.streams.OSStream|uv_pipe_t)` Stream, pipe or file +to use as stderr. {env?} `(table)` Environment variables to pass to the process {cwd?} `(string)` Current working directory of the process @@ -425,7 +492,7 @@ if an error occurred. Inherits: `nio.streams.Stream` Fields~ -{read} `(async fun(n?: integer): string,string)` Read data from the stream, +{read} `(async fun(n?: integer): string?,string?)` Read data from the stream, optionally up to n bytes otherwise until EOF is reached. Returns the data read or error message if an error occurred. @@ -433,8 +500,8 @@ or error message if an error occurred. Inherits: `nio.streams.Stream` Fields~ -{write} `(async fun(data: string): string|nil)` Write data to the stream. -Returns an error message if an error occurred. +{write} `(async fun(data: string): string?)` Write data to the stream. Returns +an error message if an error occurred. *nio.streams.OSStream* Inherits: `nio.streams.Stream` @@ -456,6 +523,10 @@ Inherits: `nio.streams.StreamWriter, nio.streams.OSStream` *nio.streams.OSStreamWriter* + *nio.streams.OSStreamReaderWriter* +Inherits: `nio.streams.OSStreamReader, nio.streams.OSStreamWriter` + + ============================================================================== nio.uv *nio.uv* diff --git a/lua/nio/file.lua b/lua/nio/file.lua new file mode 100644 index 0000000..d0fa509 --- /dev/null +++ b/lua/nio/file.lua @@ -0,0 +1,59 @@ +local uv = require("nio.uv") +local streams = require("nio.streams") + +local nio = {} + +---@class nio.file +nio.file = {} + +---@class nio.file.File : nio.streams.OSStreamReaderWriter +---@field read async fun(n: integer?, offset: integer?):string?,string? Read data from the stream, optionally up to n bytes otherwise until EOF is reached. Returns the data read or error message if an error occurred. If offset is provided, data will be read from that position in the file, otherwise the current position will be used. + +--- Open a file with the given flags and mode +--- ```lua +--- local file = nio.file.open("test.txt", "w+") +--- +--- file.write("Hello, World!\n") +--- +--- local content = file.read(nil, 0) +--- file.close() +--- print(content) +--- ``` +---@param path string The path to the file +---@param flags uv.aliases.fs_access_flags|integer? The flags to open the file with, defaults to "r" +---@param mode number? The mode to open the file with, defaults to 644 +---@return nio.file.File? File object +---@return string? Error message if an error occurred while opening +--- +---@seealso |uv.fs_open| +function nio.file.open(path, flags, mode) + local err, fd = uv.fs_open(path, flags or "r", mode or 438) + if not fd then + return nil, err + end + + local reader, reader_err = streams._file_reader(fd) + if not reader then + return nil, reader_err + end + local writer, writer_err = streams._writer(fd) + if not writer then + return nil, writer_err + end + + local file = { + fd = fd, + close = function() + local close_err = uv.fs_close(fd) + reader.close() + writer.close() + return close_err + end, + write = writer.write, + read = reader.read, + } + + return file +end + +return nio.file diff --git a/lua/nio/init.lua b/lua/nio/init.lua index 53efd67..daa959d 100644 --- a/lua/nio/init.lua +++ b/lua/nio/init.lua @@ -4,6 +4,7 @@ local control = require("nio.control") local uv = require("nio.uv") local tests = require("nio.tests") local ui = require("nio.ui") +local file = require("nio.file") local lsp = require("nio.lsp") local process = require("nio.process") @@ -28,6 +29,7 @@ nio.tests = tests nio.tasks = tasks nio.lsp = lsp nio.process = process +nio.file = file --- Run a function in an async context. This is the entrypoint to all async --- functionality. diff --git a/lua/nio/process.lua b/lua/nio/process.lua index 3222d66..7abc378 100644 --- a/lua/nio/process.lua +++ b/lua/nio/process.lua @@ -18,6 +18,17 @@ nio.process = {} --- Run a process asynchronously. --- ```lua +--- local process = nio.process.run({ +--- cmd = "printf", args = { "hello" } +--- }) +--- +--- local output = second.stdout.read() +--- print(output) +--- ``` +--- +--- Processes can be chained together, passing output of one process as input to +--- another. +--- ```lua --- local first = nio.process.run({ --- cmd = "printf", args = { "hello" } --- }) @@ -29,6 +40,23 @@ nio.process = {} --- local output = second.stdout.read() --- print(output) --- ``` +--- +--- The stdio fields can also be file objects. +--- ```lua +--- local path = nio.fn.tempname() +--- +--- local file = nio.file.open(path, "w+") +--- +--- local process = nio.process.run({ +--- cmd = "printf", +--- args = { "hello" }, +--- stdout = file, +--- }) +--- process.result() +--- +--- local output = file.read(nil, 0) +--- print(output) +--- ``` ---@param opts nio.process.RunOpts ---@return nio.process.Process? Process object for the running process ---@return string? Error message if an error occurred @@ -40,15 +68,15 @@ function nio.process.run(opts) local exit_code_future = control.future() - local stdout, stdout_err = streams.reader(opts.stdout) + local stdout, stdout_err = streams._socket_reader(opts.stdout) if not stdout then return nil, stdout_err end - local stderr, stderr_err = streams.reader(opts.stderr) + local stderr, stderr_err = streams._socket_reader(opts.stderr) if not stderr then return nil, stderr_err end - local stdin, stdin_err = streams.writer(opts.stdin) + local stdin, stdin_err = streams._writer(opts.stdin) if not stdin then return nil, stdin_err end @@ -114,12 +142,12 @@ end ---@class nio.process.RunOpts ---@field cmd string Command to run ---@field args? string[] Arguments to pass to the command ----@field stdin? integer|nio.streams.OSStreamReader|uv.uv_pipe_t|uv_pipe_t Stream, ---- pipe or file descriptor to use as stdin. ----@field stdout? integer|nio.streams.OSStreamWriter|uv.uv_pipe_t|uv_pipe_t Stream, ---- pipe or file descriptor to use as stdout. ----@field stderr? integer|nio.streams.OSStreamWriter|uv.uv_pipe_t|uv_pipe_t Stream, ---- pipe or file descriptor to use as stderr. +---@field stdin? integer|nio.streams.OSStream|uv_pipe_t Stream, pipe or file +---descriptor to use as stdin. +---@field stdout? integer|nio.streams.OSStream|uv_pipe_t Stream, pipe or file +---descriptor to use as stdout. +---@field stderr? integer|nio.streams.OSStream|uv_pipe_t Stream, pipe or file +---descriptor to use as stderr. ---@field env? table Environment variables to pass to the --- process ---@field cwd? string Current working directory of the process diff --git a/lua/nio/streams.lua b/lua/nio/streams.lua index dcc0a79..5c6ce87 100644 --- a/lua/nio/streams.lua +++ b/lua/nio/streams.lua @@ -12,10 +12,10 @@ nio.streams = {} ---@field close async fun(): string|nil Close the stream. Returns an error message if an error occurred. ---@class nio.streams.Reader : nio.streams.Stream ----@field read async fun(n?: integer): string,string Read data from the stream, optionally up to n bytes otherwise until EOF is reached. Returns the data read or error message if an error occurred. +---@field read async fun(n?: integer): string?,string? Read data from the stream, optionally up to n bytes otherwise until EOF is reached. Returns the data read or error message if an error occurred. ---@class nio.streams.Writer : nio.streams.Stream ----@field write async fun(data: string): string|nil Write data to the stream. Returns an error message if an error occurred. +---@field write async fun(data: string): string? Write data to the stream. Returns an error message if an error occurred. ---@class nio.streams.OSStream : nio.streams.Stream ---@field fd integer The file descriptor of the stream @@ -26,6 +26,8 @@ nio.streams = {} ---@class nio.streams.OSStreamReader : nio.streams.StreamReader, nio.streams.OSStream ---@class nio.streams.OSStreamWriter : nio.streams.StreamWriter, nio.streams.OSStream +---@class nio.streams.OSStreamReaderWriter : nio.streams.OSStreamReader, nio.streams.OSStreamWriter + ---@param input integer|uv.uv_pipe_t|uv_pipe_t|nio.streams.OSStream ---@return uv_pipe_t? ---@return string? @@ -53,17 +55,17 @@ local function create_pipe(input) end ---@param input integer|nio.streams.OSStreamReader|uv.uv_pipe_t|uv_pipe_t ----@return {pipe: uv_pipe_t, read: (fun(n?: integer):string,string), close: fun(): string|nil}|nil ----@return string|nil ----@private -function nio.streams.reader(input) +---@return {pipe: uv_pipe_t, read: (fun(n?: integer):string,string), close: fun(): string|nil}? +---@return string? +---@nodoc +function nio.streams._socket_reader(input) local pipe, create_err = create_pipe(input) if not pipe then return nil, create_err end local buffer = "" - local ready = control.event() + local has_buffer = control.event() local complete = control.event() local started = false @@ -73,24 +75,39 @@ function nio.streams.reader(input) end vim.loop.read_stop(pipe) complete.set() - ready.set() + has_buffer.set() end local read_err = nil local start = function() started = true + local fd, fd_err = pipe:fileno() + if not fd then + return fd_err + end + local stat_err, stat = uv.fs_fstat(fd) + if stat_err or not stat then + return stat_err + end + + if stat.type ~= "socket" then + return "Invalid pipe type, expected socket, got " .. stat.type + end + local _, read_start_err = pipe:read_start(function(err, data) if err then read_err = err - ready.set() + has_buffer.set() return end + if not data then tasks.run(stop_reading) return end + buffer = buffer .. data - ready.set() + has_buffer.set() end) return read_start_err end @@ -111,10 +128,9 @@ function nio.streams.reader(input) if n == 0 then return "", nil end - while not complete.is_set() and (not n or #buffer < n) and not read_err do - ready.wait() - ready.clear() + has_buffer.wait() + has_buffer.clear() end if read_err then @@ -128,11 +144,37 @@ function nio.streams.reader(input) } end +---@param fd integer +---@return {read: (fun(n: integer?, offset: integer?): string?,string?), close: (async fun(): string?), seek: fun(offset: integer, whence: string?)}? +---@nodoc +function nio.streams._file_reader(fd) + return { + close = function() + return uv.fs_close(fd) + end, + read = function(n, offset) + if n == 0 then + return "", nil + end + if not n then + local stat_err, stat = uv.fs_fstat(fd) + if stat_err or not stat then + return nil, stat_err + end + n = stat.size + end + + local read_err, data = uv.fs_read(fd, n, offset) + return data, read_err + end, + } +end + ---@param input integer|nio.streams.OSStreamWriter|uv.uv_pipe_t|uv_pipe_t ---@return {pipe: uv_pipe_t, write: (fun(data: string): string|nil), close: fun(): string|nil}|nil ---@return string|nil ----@private -function nio.streams.writer(input) +---@nodoc +function nio.streams._writer(input) local pipe, create_err = create_pipe(input) if not pipe then return nil, create_err @@ -143,12 +185,12 @@ function nio.streams.writer(input) write = function(data) local maybe_err = uv.write(pipe, data) if type(maybe_err) == "string" then - return maybe_err + return vim.loop.translate_sys_error(vim.loop.errno[maybe_err]) end return nil end, close = function() - return uv.shutdown(pipe) + uv.close(pipe) end, } end diff --git a/lua/nio/tasks.lua b/lua/nio/tasks.lua index 9291bf1..fd05f8e 100644 --- a/lua/nio/tasks.lua +++ b/lua/nio/tasks.lua @@ -173,8 +173,8 @@ end ---@package ---@field opts nio.WrapOpts ---@nodoc -function nio.tasks.wrap(func, argc, args) - args = vim.tbl_extend("keep", args or {}, { strict = true }) +function nio.tasks.wrap(func, argc, opts) + opts = vim.tbl_extend("keep", opts or {}, { strict = true }) vim.validate({ func = { func, "function" }, argc = { argc, "number" } }) local protected = function(...) local args = { ... } @@ -189,7 +189,7 @@ function nio.tasks.wrap(func, argc, args) return function(...) if not current_non_main_co() then - if args.strict then + if opts.strict then error("Cannot call async function from non-async context") end return func(...) diff --git a/scripts/gendocs.lua b/scripts/gendocs.lua index 2a5351d..a3a04be 100644 --- a/scripts/gendocs.lua +++ b/scripts/gendocs.lua @@ -843,6 +843,7 @@ minidoc.generate( "./lua/nio/init.lua", "./lua/nio/control.lua", "./lua/nio/lsp.lua", + "./lua/nio/file.lua", "./lua/nio/process.lua", "./lua/nio/streams.lua", "./lua/nio/uv.lua", diff --git a/tests/file_spec.lua b/tests/file_spec.lua new file mode 100644 index 0000000..8220906 --- /dev/null +++ b/tests/file_spec.lua @@ -0,0 +1,101 @@ +local nio = require("nio") +local a = nio.tests + +describe("file", function() + a.it("opens file", function() + local path = assert(nio.fn.tempname()) + + local file = assert(nio.file.open(path, "w")) + + assert.True(file.fd > 0) + end) + + a.it("returns opening error", function() + local path = assert(nio.fn.tempname()) + + local file, open_err = nio.file.open(path, "r") + + assert.equal(open_err, "ENOENT: no such file or directory: " .. path) + assert.Nil(file) + end) + + a.it("writes file", function() + local path = assert(nio.fn.tempname()) + + local file = assert(nio.file.open(path, "w")) + + local err = file.write("hello") + + local io_file = assert(io.open(path, "r")) + local content = io_file:read() + io_file:close() + + assert.Nil(err) + assert.equal(content, "hello") + end) + + a.it("return error writing", function() + local path = assert(nio.fn.tempname()) + + local file = assert(nio.file.open(path, "w")) + + nio.uv.fs_close(file.fd) + + local err = file.write("hello") + + assert.equal("EBADF: bad file descriptor", err) + end) + + a.it("reads file", function() + local path = assert(nio.fn.tempname()) + local io_file = assert(io.open(path, "w")) + io_file:write("hello") + io_file:close() + + local file = assert(nio.file.open(path)) + local content, err = file.read() + + assert.Nil(err) + assert.equal("hello", content) + end) + + a.it("reads file up to n", function() + local path = assert(nio.fn.tempname()) + local io_file = assert(io.open(path, "w")) + io_file:write("hello") + io_file:close() + + local file = assert(nio.file.open(path)) + local content, err = file.read(3) + + assert.Nil(err) + assert.equal("hel", content) + end) + + a.it("reads file from offset", function() + local path = assert(nio.fn.tempname()) + local io_file = assert(io.open(path, "w")) + io_file:write("hello") + io_file:close() + + local file = assert(nio.file.open(path)) + local content, err = file.read(nil, 1) + + assert.Nil(err) + assert.equal("ello", content) + end) + + a.it("returns error when reading", function() + local path = assert(nio.fn.tempname()) + local io_file = assert(io.open(path, "w")) + io_file:write("hello") + io_file:close() + + local file = assert(nio.file.open(path)) + nio.uv.fs_close(file.fd) + local content, err = file.read() + + assert.Nil(content) + assert.equal("EBADF: bad file descriptor", err) + end) +end) diff --git a/tests/process_spec.lua b/tests/process_spec.lua index 9d76390..fb0805b 100644 --- a/tests/process_spec.lua +++ b/tests/process_spec.lua @@ -84,6 +84,42 @@ describe("process", function() assert.equal(output, "hello") end) + a.it("pipes from file", function() + local path = assert(nio.fn.tempname()) + + local write_file = assert(nio.file.open(path, "w+")) + + write_file.write("hello") + write_file.close() + + local read_file = assert(nio.file.open(path, "r")) + + local process = assert(nio.process.run({ + cmd = "cat", + stdin = read_file, + })) + process.result() + local output = process.stdout.read() + assert.equal(output, "hello") + end) + + a.it("pipes to file", function() + local path = assert(nio.fn.tempname()) + + local file = assert(nio.file.open(path, "w+")) + + local process = assert(nio.process.run({ + cmd = "cat", + stdout = file, + })) + process.stdin.write("hello") + process.stdin.close() + process.result() + + local output = file.read(nil, 0) + assert.equal(output, "hello") + end) + a.it("reads input from uv_pipe_t", function() local pipe = assert(vim.loop.new_pipe())