first commit of lua-mmapfile

.gitignore

@@ -0,0 +1,3 @@
+.DS_store
+doc
+lua/mmapfile-test

AUTHORS

@@ -0,0 +1 @@
+Leyland, Geoff

LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2014 Incremental IP Limited
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,59 @@
+# Lua-mmapfile - A simple interface for mmaping files
+
+## 1. What?
+
+mmapfile uses `mmap` to provide a way of quickly storing and loading data
+that's already in some kind of in-memory binary format.
+
+`create` creates a new file and maps new memory to that file.  You can
+then write to the memory to write to the file.
+
+`open` opens an existing file and maps its contents to memory, returning
+a pointer to the memory and the length of the file.
+
+`close` syncs memory to the file, closes the file, and deletes the
+mapping between the memory and the file.
+
+The "gc" variants of `create` and `open` (`gccreate` and `gcopen`) set
+up a garbage collection callback for the pointer so that the file is
+correctly closed when the pointer is no longer referenced.  Not
+appropriate if you might be storing the pointer in C, referencing it from
+unmanaged memory, or casting it to another type!
+
+All memory is mapped above 4G to try to keep away from the memory space
+LuaJIT uses.
+
+
+## 2. How?
+
+    local ffi = require"ffi"
+
+    ffi.cdef"struct test { int a; double b; };"
+
+    local mmapfile = require"mmapfile"
+
+    local ptr1 = mmapfile.gccreate("mmapfile-test", 1, "struct test")
+    ptr1.a = 1
+    ptr1.b = 1.5
+    ptr1 = nil
+    collectgarbage()
+
+    local ptr2, size = mmapfile.gcopen("mmapfile-test", "struct test")
+    assert(size == 1)
+    assert(ptr2.a == 1)
+    assert(ptr2.b == 1.5)
+
+For more details `make doc` or `ldoc lua --all`.
+
+
+## 3. Requirements
+
+[LuaJIT](http://luajit.org) and
+[ljsyscall](https://github.com/justincormack/ljsyscall)
+
+
+## 4. Issues
+
++ Should probably have an option for directly mapping existing memory, but
+  I don't know enough about page boundaries.
++ No windows support

lua/config.ld

@@ -0,0 +1,5 @@
+project = "Lua-mmapfile"
+title = "Lua-mmapfile"
+description = "Lua-mmapfile provides a simple interface to mmap"
+format = "markdown"
+all = true

lua/mmapfile.lua

@@ -0,0 +1,200 @@
+--- A simple interface to mmap.
+--  mmapfile provides a way of quickly storing and loading data that's
+--  already in some kind of in-memory binary format.
+--
+--  `create` creates a new file and maps new memory to that file.  You can
+--  then write to the memory to write to the file.
+--
+--  `open` opens an existing file and maps its contents to memory, returning
+--  a pointer to the memory and the length of the file.
+--
+--  `close` syncs memory to the file, closes the file, and deletes the
+--  mapping between the memory and the file.
+--
+--  The "gc" variants of `create` and `open` (`gccreate` and `gcopen`) set
+--  up a garbage collection callback for the pointer so that the file is
+--  correctly closed when the pointer is no longer referenced.  Not
+--  appropriate if you might be storing the pointer in C, referencing it from
+--  unmanaged memory, or casting it to another type!
+--
+--  All memory is mapped above 4G to try to keep away from the memory space
+--  LuaJIT uses.
+
+local S = require"syscall"
+local ffi = require"ffi"
+
+
+------------------------------------------------------------------------------
+
+local function assert(condition, message)
+  if condition then return condition end
+  message = message or "assertion failed"
+  error(tostring(message), 2)
+end
+
+
+------------------------------------------------------------------------------
+
+--- Call mmap until we get an address higher that 4 gigabytes.
+--  mmapping over 4G means we don't step on LuaJIT's toes, and this usually
+--  works first time.
+--  See `man mmap` for explanation of parameters.
+--  @treturn pointer: the memory allocated.
+local function mmap_4G(
+  size,         -- integer: size to allocate in bytes
+  fd,           -- integer: file descriptor to map to
+  prot,         -- string: mmap's prot, as interpreted by syscall
+  flags)        -- string: mmap's flags, as interpreted by syscall
+
+  local base, step = 4 * 1024 * 1024 * 1024, 2^math.floor(math.log(tonumber(size)) / math.log(2))
+  local addr
+  while true do
+    addr = S.mmap(ffi.cast("void*", base), size, prot, flags, fd, 0)
+    if addr >= ffi.cast("void*", 4 * 1024 * 1024 * 1024) then break end
+    S.munmap(addr, size)
+    base = base + step
+  end
+  return addr
+end
+
+
+------------------------------------------------------------------------------
+
+local open_fds = {}
+
+
+--- Close a mapping between a file and an address.
+--  `msync` the memory to its associated file, `munmap` the memory, and close
+--  the file.
+local function close(
+  addr)         -- pointer: the mapped address to unmap.
+  local s = tostring(ffi.cast("void*", addr))
+  local fd = assert(open_fds[s], "no file open for this address")
+  open_fds[s] = nil
+
+  -- it seems that file descriptors get closed before final __gc calls in
+  -- some exit scenarios, so we don't worry too much if we can't
+  -- stat the fd
+  local st = fd:stat()
+  if st then
+    assert(S.msync(addr, st.size, "sync"))
+    assert(S.munmap(addr, st.size))
+    assert(fd:close())
+  end
+end
+
+
+--- Allocate memory and create a new file mapped to it.
+--  Use create to set aside an area of memory to write to a file.
+--  If `type` is supplied then the pointer to the allocated memory is cast
+--  to the correct type, and `size` is the number of `type`, not bytes,
+--  to allocate.
+--  If `data` is supplied, then the data at `data` is copied into the mapped
+--  memory (and so written to the file).  It might make more sense just to
+--  map the pointer `data` directly to the file, but that might require `data`
+--  to be on a page boundary.
+--  The file descriptor is saved in a table keyed to the address allocated
+--  so that close can find the write fd to close when the memory is unmapped.
+--  @treturn pointer: the memory allocated.
+local function create(
+  filename,     -- string: name of the file to create.
+  size,         -- integer: number of bytes or `type`s to allocate.
+  type,         -- ?string: type to allocate
+  data)         -- ?pointer: data to copy to the mapped area.
+  local fd = assert(S.open(filename, "RDWR, CREAT", "RUSR, WUSR, RGRP, ROTH"))
+
+  if type then
+    size = size * ffi.sizeof(type)
+  end
+
+  assert(fd:lseek(size-1, "set"))
+  assert(fd:write(ffi.new("char[1]", 0), 1))
+
+  local addr = assert(mmap_4G(size, fd, "read, write", "file, shared"))
+
+  open_fds[tostring(ffi.cast("void*", addr))] = fd
+
+  if data then
+    ffi.copy(addr, data, size)
+  end
+
+  if type then
+    return ffi.cast(type.."*", addr)
+  else
+    return addr
+  end
+end
+
+
+--- Same as create, but set up a GC cleanup for the memory and file.
+--  @treturn pointer: the memory allocated
+local function gccreate(
+  filename,     -- string: name of the file to create.
+  size,         -- integer: number of bytes or `type`s to allocate.
+  type,         -- ?string: type to allocate
+  data)         -- ?pointer: data to copy to the mapped area.
+  return ffi.gc(create(filename, size, type, data), close)
+end
+
+
+--- Map an existing file to an area of memory.
+--  If `type` is present, the the pointer returned is cast to the `type*` and
+--  the size returned is the number of `types`, not bytes.
+--  @treturn pointer: the memory allocated.
+--  @treturn int: size of the file, in bytes or `type`s.
+local function open(
+  filename,     -- string: name of the file to open.
+  type,         -- ?string: type to allocate
+  mode)         -- ?string: open mode for the file "r" or "rw"
+  mode = mode or "r"
+  local filemode, mapmode
+  if mode == "r" then
+    filemode = "rdonly"
+    mapmode = "read"
+  elseif mode == "rw" then
+    filemode = "rdwr"
+    mapmode = "read, write"
+  else
+    return nil, "unknown read/write mode"
+  end
+
+  local fd = assert(S.open(filename, filemode, 0))
+  local st = assert(fd:stat())
+
+  local addr = assert(mmap_4G(st.size, fd, mapmode, "file, shared"))
+
+  open_fds[tostring(ffi.cast("void*", addr))] = fd
+
+  if type then
+    return ffi.cast(type.."*", addr), st.size / ffi.sizeof(type)
+  else
+    return addr, st.size
+  end
+end
+
+
+--- Same as open, but set up a GC cleanup for the memory and file.
+--  @treturn pointer: the memory allocated.
+--  @treturn int: size of the file, in bytes or `type`s.
+local function gcopen(
+  filename,     -- string: name of the file to open.
+  type,         -- ?string: type to allocate
+  mode)         -- ?string: open mode for the file "r" or "rw"
+  local addr, size = open(filename, type, mode)
+  return ffi.gc(addr, close), size
+end
+
+
+------------------------------------------------------------------------------
+
+return
+{
+  create = create,
+  gccreate = gccreate,
+  open = open,
+  gcopen = gcopen,
+  close = close,
+}
+
+------------------------------------------------------------------------------
+

lua/test/test.lua

@@ -0,0 +1,18 @@
+local ffi = require"ffi"
+
+ffi.cdef"struct test { int a; double b; };"
+
+local mmapfile = require"mmapfile"
+
+local ptr1 = mmapfile.gccreate("mmapfile-test", 1, "struct test")
+ptr1.a = 1
+ptr1.b = 1.5
+ptr1 = nil
+collectgarbage()
+
+local ptr2, size = mmapfile.gcopen("mmapfile-test", "struct test")
+assert(size == 1)
+assert(ptr2.a == 1)
+assert(ptr2.b == 1.5)
+
+io.stderr:write("Test passed\n")

makefile

@@ -0,0 +1,17 @@
+LUA= $(shell echo `which lua`)
+LUA_BINDIR= $(shell echo `dirname $(LUA)`)
+LUA_PREFIX= $(shell echo `dirname $(LUA_BINDIR)`)
+LUA_VERSION = $(shell echo `lua -v 2>&1 | cut -d " " -f 2 | cut -b 1-3`)
+LUA_SHAREDIR=$(LUA_PREFIX)/share/lua/$(LUA_VERSION)
+
+default:
+	@echo "Nothing to build.  Try 'make install'."
+
+install:
+	cp lua/mmapfile.lua $(LUA_SHAREDIR)
+
+doc: lua/mmapfile.lua lua/config.ld
+	ldoc lua --all
+
+test:
+	cd lua && luajit test/test.lua

rockspecs/mmapfile-1-1.rockspec

@@ -0,0 +1,28 @@
+package = "mmapfile"
+version = "1-1"
+source =
+{
+  url = "git://github.com/geoffleyland/lua-mmapfile.git",
+  branch = "master",
+  tag = "v1",
+}
+description =
+{
+  summary = "Simple memory-mapped files",
+  homepage = "http://github.com/geoffleyland/lua-mmapfile",
+  license = "MIT/X11",
+  maintainer = "Geoff Leyland <geoff.leyland@incremental.co.nz>"
+}
+dependencies =
+{
+    'lua == 5.1',               -- In fact this should be "luajit >= 2.0.0"
+    'ljsyscall >= 0.9',
+}
+build =
+{
+  type = "builtin",
+  modules =
+  {
+    mmapfile = "lua/mmapfile.lua",
+  },
+}

rockspecs/mmapfile-scm-1.rockspec

@@ -0,0 +1,27 @@
+package = "mmapfile"
+version = "scm-1"
+source =
+{
+  url = "git://github.com/geoffleyland/lua-mmapfile.git",
+  branch = "master",
+}
+description =
+{
+  summary = "Simple memory-mapped files",
+  homepage = "http://github.com/geoffleyland/lua-mmapfile",
+  license = "MIT/X11",
+  maintainer = "Geoff Leyland <geoff.leyland@incremental.co.nz>"
+}
+dependencies =
+{
+    'lua == 5.1',               -- should be "luajit >= 2.0.0"
+    'ljsyscall >= 0.9',
+}
+build =
+{
+  type = "builtin",
+  modules =
+  {
+    mmapfile = "lua/mmapfile.lua",
+  },
+}