From ee252454be6c60f4a35842383fcb4e36830eb412 Mon Sep 17 00:00:00 2001 From: Geoff Leyland Date: Fri, 24 Jan 2014 11:37:33 +1300 Subject: [PATCH] first commit of lua-mmapfile --- .gitignore | 3 + AUTHORS | 1 + LICENSE | 19 +++ README.md | 59 +++++++++ lua/config.ld | 5 + lua/mmapfile.lua | 200 ++++++++++++++++++++++++++++++ lua/test/test.lua | 18 +++ makefile | 17 +++ rockspecs/mmapfile-1-1.rockspec | 28 +++++ rockspecs/mmapfile-scm-1.rockspec | 27 ++++ 10 files changed, 377 insertions(+) create mode 100644 .gitignore create mode 100644 AUTHORS create mode 100644 LICENSE create mode 100644 README.md create mode 100644 lua/config.ld create mode 100644 lua/mmapfile.lua create mode 100644 lua/test/test.lua create mode 100644 makefile create mode 100644 rockspecs/mmapfile-1-1.rockspec create mode 100644 rockspecs/mmapfile-scm-1.rockspec diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..35ec585 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.DS_store +doc +lua/mmapfile-test diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..bf6696b --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Leyland, Geoff \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..33220c1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2014 Incremental IP Limited + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..86133ac --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +# Lua-mmapfile - A simple interface for mmaping files + +## 1. What? + +mmapfile uses `mmap` to provide a way of quickly storing and loading data +that's already in some kind of in-memory binary format. + +`create` creates a new file and maps new memory to that file. You can +then write to the memory to write to the file. + +`open` opens an existing file and maps its contents to memory, returning +a pointer to the memory and the length of the file. + +`close` syncs memory to the file, closes the file, and deletes the +mapping between the memory and the file. + +The "gc" variants of `create` and `open` (`gccreate` and `gcopen`) set +up a garbage collection callback for the pointer so that the file is +correctly closed when the pointer is no longer referenced. Not +appropriate if you might be storing the pointer in C, referencing it from +unmanaged memory, or casting it to another type! + +All memory is mapped above 4G to try to keep away from the memory space +LuaJIT uses. + + +## 2. How? + + local ffi = require"ffi" + + ffi.cdef"struct test { int a; double b; };" + + local mmapfile = require"mmapfile" + + local ptr1 = mmapfile.gccreate("mmapfile-test", 1, "struct test") + ptr1.a = 1 + ptr1.b = 1.5 + ptr1 = nil + collectgarbage() + + local ptr2, size = mmapfile.gcopen("mmapfile-test", "struct test") + assert(size == 1) + assert(ptr2.a == 1) + assert(ptr2.b == 1.5) + +For more details `make doc` or `ldoc lua --all`. + + +## 3. Requirements + +[LuaJIT](http://luajit.org) and +[ljsyscall](https://github.com/justincormack/ljsyscall) + + +## 4. Issues + ++ Should probably have an option for directly mapping existing memory, but + I don't know enough about page boundaries. ++ No windows support diff --git a/lua/config.ld b/lua/config.ld new file mode 100644 index 0000000..6d18e4c --- /dev/null +++ b/lua/config.ld @@ -0,0 +1,5 @@ +project = "Lua-mmapfile" +title = "Lua-mmapfile" +description = "Lua-mmapfile provides a simple interface to mmap" +format = "markdown" +all = true diff --git a/lua/mmapfile.lua b/lua/mmapfile.lua new file mode 100644 index 0000000..fd3508b --- /dev/null +++ b/lua/mmapfile.lua @@ -0,0 +1,200 @@ +--- A simple interface to mmap. +-- mmapfile provides a way of quickly storing and loading data that's +-- already in some kind of in-memory binary format. +-- +-- `create` creates a new file and maps new memory to that file. You can +-- then write to the memory to write to the file. +-- +-- `open` opens an existing file and maps its contents to memory, returning +-- a pointer to the memory and the length of the file. +-- +-- `close` syncs memory to the file, closes the file, and deletes the +-- mapping between the memory and the file. +-- +-- The "gc" variants of `create` and `open` (`gccreate` and `gcopen`) set +-- up a garbage collection callback for the pointer so that the file is +-- correctly closed when the pointer is no longer referenced. Not +-- appropriate if you might be storing the pointer in C, referencing it from +-- unmanaged memory, or casting it to another type! +-- +-- All memory is mapped above 4G to try to keep away from the memory space +-- LuaJIT uses. + +local S = require"syscall" +local ffi = require"ffi" + + +------------------------------------------------------------------------------ + +local function assert(condition, message) + if condition then return condition end + message = message or "assertion failed" + error(tostring(message), 2) +end + + +------------------------------------------------------------------------------ + +--- Call mmap until we get an address higher that 4 gigabytes. +-- mmapping over 4G means we don't step on LuaJIT's toes, and this usually +-- works first time. +-- See `man mmap` for explanation of parameters. +-- @treturn pointer: the memory allocated. +local function mmap_4G( + size, -- integer: size to allocate in bytes + fd, -- integer: file descriptor to map to + prot, -- string: mmap's prot, as interpreted by syscall + flags) -- string: mmap's flags, as interpreted by syscall + + local base, step = 4 * 1024 * 1024 * 1024, 2^math.floor(math.log(tonumber(size)) / math.log(2)) + local addr + while true do + addr = S.mmap(ffi.cast("void*", base), size, prot, flags, fd, 0) + if addr >= ffi.cast("void*", 4 * 1024 * 1024 * 1024) then break end + S.munmap(addr, size) + base = base + step + end + return addr +end + + +------------------------------------------------------------------------------ + +local open_fds = {} + + +--- Close a mapping between a file and an address. +-- `msync` the memory to its associated file, `munmap` the memory, and close +-- the file. +local function close( + addr) -- pointer: the mapped address to unmap. + local s = tostring(ffi.cast("void*", addr)) + local fd = assert(open_fds[s], "no file open for this address") + open_fds[s] = nil + + -- it seems that file descriptors get closed before final __gc calls in + -- some exit scenarios, so we don't worry too much if we can't + -- stat the fd + local st = fd:stat() + if st then + assert(S.msync(addr, st.size, "sync")) + assert(S.munmap(addr, st.size)) + assert(fd:close()) + end +end + + +--- Allocate memory and create a new file mapped to it. +-- Use create to set aside an area of memory to write to a file. +-- If `type` is supplied then the pointer to the allocated memory is cast +-- to the correct type, and `size` is the number of `type`, not bytes, +-- to allocate. +-- If `data` is supplied, then the data at `data` is copied into the mapped +-- memory (and so written to the file). It might make more sense just to +-- map the pointer `data` directly to the file, but that might require `data` +-- to be on a page boundary. +-- The file descriptor is saved in a table keyed to the address allocated +-- so that close can find the write fd to close when the memory is unmapped. +-- @treturn pointer: the memory allocated. +local function create( + filename, -- string: name of the file to create. + size, -- integer: number of bytes or `type`s to allocate. + type, -- ?string: type to allocate + data) -- ?pointer: data to copy to the mapped area. + local fd = assert(S.open(filename, "RDWR, CREAT", "RUSR, WUSR, RGRP, ROTH")) + + if type then + size = size * ffi.sizeof(type) + end + + assert(fd:lseek(size-1, "set")) + assert(fd:write(ffi.new("char[1]", 0), 1)) + + local addr = assert(mmap_4G(size, fd, "read, write", "file, shared")) + + open_fds[tostring(ffi.cast("void*", addr))] = fd + + if data then + ffi.copy(addr, data, size) + end + + if type then + return ffi.cast(type.."*", addr) + else + return addr + end +end + + +--- Same as create, but set up a GC cleanup for the memory and file. +-- @treturn pointer: the memory allocated +local function gccreate( + filename, -- string: name of the file to create. + size, -- integer: number of bytes or `type`s to allocate. + type, -- ?string: type to allocate + data) -- ?pointer: data to copy to the mapped area. + return ffi.gc(create(filename, size, type, data), close) +end + + +--- Map an existing file to an area of memory. +-- If `type` is present, the the pointer returned is cast to the `type*` and +-- the size returned is the number of `types`, not bytes. +-- @treturn pointer: the memory allocated. +-- @treturn int: size of the file, in bytes or `type`s. +local function open( + filename, -- string: name of the file to open. + type, -- ?string: type to allocate + mode) -- ?string: open mode for the file "r" or "rw" + mode = mode or "r" + local filemode, mapmode + if mode == "r" then + filemode = "rdonly" + mapmode = "read" + elseif mode == "rw" then + filemode = "rdwr" + mapmode = "read, write" + else + return nil, "unknown read/write mode" + end + + local fd = assert(S.open(filename, filemode, 0)) + local st = assert(fd:stat()) + + local addr = assert(mmap_4G(st.size, fd, mapmode, "file, shared")) + + open_fds[tostring(ffi.cast("void*", addr))] = fd + + if type then + return ffi.cast(type.."*", addr), st.size / ffi.sizeof(type) + else + return addr, st.size + end +end + + +--- Same as open, but set up a GC cleanup for the memory and file. +-- @treturn pointer: the memory allocated. +-- @treturn int: size of the file, in bytes or `type`s. +local function gcopen( + filename, -- string: name of the file to open. + type, -- ?string: type to allocate + mode) -- ?string: open mode for the file "r" or "rw" + local addr, size = open(filename, type, mode) + return ffi.gc(addr, close), size +end + + +------------------------------------------------------------------------------ + +return +{ + create = create, + gccreate = gccreate, + open = open, + gcopen = gcopen, + close = close, +} + +------------------------------------------------------------------------------ + diff --git a/lua/test/test.lua b/lua/test/test.lua new file mode 100644 index 0000000..5808e2c --- /dev/null +++ b/lua/test/test.lua @@ -0,0 +1,18 @@ +local ffi = require"ffi" + +ffi.cdef"struct test { int a; double b; };" + +local mmapfile = require"mmapfile" + +local ptr1 = mmapfile.gccreate("mmapfile-test", 1, "struct test") +ptr1.a = 1 +ptr1.b = 1.5 +ptr1 = nil +collectgarbage() + +local ptr2, size = mmapfile.gcopen("mmapfile-test", "struct test") +assert(size == 1) +assert(ptr2.a == 1) +assert(ptr2.b == 1.5) + +io.stderr:write("Test passed\n") \ No newline at end of file diff --git a/makefile b/makefile new file mode 100644 index 0000000..119ef0f --- /dev/null +++ b/makefile @@ -0,0 +1,17 @@ +LUA= $(shell echo `which lua`) +LUA_BINDIR= $(shell echo `dirname $(LUA)`) +LUA_PREFIX= $(shell echo `dirname $(LUA_BINDIR)`) +LUA_VERSION = $(shell echo `lua -v 2>&1 | cut -d " " -f 2 | cut -b 1-3`) +LUA_SHAREDIR=$(LUA_PREFIX)/share/lua/$(LUA_VERSION) + +default: + @echo "Nothing to build. Try 'make install'." + +install: + cp lua/mmapfile.lua $(LUA_SHAREDIR) + +doc: lua/mmapfile.lua lua/config.ld + ldoc lua --all + +test: + cd lua && luajit test/test.lua \ No newline at end of file diff --git a/rockspecs/mmapfile-1-1.rockspec b/rockspecs/mmapfile-1-1.rockspec new file mode 100644 index 0000000..047375e --- /dev/null +++ b/rockspecs/mmapfile-1-1.rockspec @@ -0,0 +1,28 @@ +package = "mmapfile" +version = "1-1" +source = +{ + url = "git://github.com/geoffleyland/lua-mmapfile.git", + branch = "master", + tag = "v1", +} +description = +{ + summary = "Simple memory-mapped files", + homepage = "http://github.com/geoffleyland/lua-mmapfile", + license = "MIT/X11", + maintainer = "Geoff Leyland " +} +dependencies = +{ + 'lua == 5.1', -- In fact this should be "luajit >= 2.0.0" + 'ljsyscall >= 0.9', +} +build = +{ + type = "builtin", + modules = + { + mmapfile = "lua/mmapfile.lua", + }, +} diff --git a/rockspecs/mmapfile-scm-1.rockspec b/rockspecs/mmapfile-scm-1.rockspec new file mode 100644 index 0000000..300bbea --- /dev/null +++ b/rockspecs/mmapfile-scm-1.rockspec @@ -0,0 +1,27 @@ +package = "mmapfile" +version = "scm-1" +source = +{ + url = "git://github.com/geoffleyland/lua-mmapfile.git", + branch = "master", +} +description = +{ + summary = "Simple memory-mapped files", + homepage = "http://github.com/geoffleyland/lua-mmapfile", + license = "MIT/X11", + maintainer = "Geoff Leyland " +} +dependencies = +{ + 'lua == 5.1', -- should be "luajit >= 2.0.0" + 'ljsyscall >= 0.9', +} +build = +{ + type = "builtin", + modules = + { + mmapfile = "lua/mmapfile.lua", + }, +}