Preliminary windows support and tests

.busted

@@ -0,0 +1,16 @@
+return
+{
+  default =
+  {
+    verbose = true,
+    lpath   = "./lua/?.lua",
+    ROOT    = "./spec",
+    output  = "utfTerminal",
+  },
+  travis =
+  {
+    lpath   = "./lua/?.lua",
+    ROOT    = "./spec",
+    output  = "TAP",
+  }
+}

.gitignore

@@ -1,3 +1,5 @@
 .DS_store
 doc
-lua/mmapfile-test
+test
+luacov.report.out
+luacov.stats.out

.travis.yml

@@ -0,0 +1,26 @@
+language: python
+sudo: false
+
+env:
+  - LUA='luajit=2.0'
+  - LUA='luajit=2.1' 
+
+before_install:
+  - pip install hererocks
+  - hererocks lua_install -r^ --$LUA
+  - export PATH=$PATH:$PWD/lua_install/bin
+
+
+install:
+  - luarocks install luacheck
+  - luarocks install busted
+  - luarocks install luacov
+  - luarocks install luacov-coveralls
+
+script:
+  - luacheck lua
+  - luacheck --std max+busted spec
+  - busted --run=travis --coverage .
+
+after_success:
+  - luacov-coveralls -i lua

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014 Incremental IP Limited
+Copyright (c) 2014-2016 Incremental IP Limited
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

lua/mmapfile.lua

@@ -1,6 +1,7 @@
---- A simple interface to mmap.
---  mmapfile uses `mmap` to provide a way of quickly storing and loading data
---  that's already in some kind of in-memory binary format.
+--- A simple interface to mmap or MapViewOfFile
+--  mmapfile uses `mmap` (on Unix) or `MapViewOfFileEx` (on Windows)  to
+--  provide a way of quickly storing and loading data that's already in some
+--  kind of in-memory binary format.
 --
 --  `create` creates a new file and maps new memory to that file.  You can
 --  then write to the memory to write to the file.
@@ -20,232 +21,33 @@
 --  All memory is mapped above 4G to try to keep away from the memory space
 --  LuaJIT uses.
 
-local S = require"syscall"
 local ffi = require"ffi"
+local platform_mmapfile = (jit.os == "Windows" and require"mmapfile.windows") or
+                          require"mmapfile.unix"
 
 
 ------------------------------------------------------------------------------
 
-local function assert(condition, message)
-  if condition then return condition end
-  message = message or "assertion failed"
-  error(tostring(message), 2)
-end
-
-
-------------------------------------------------------------------------------
-
---- Call mmap until we get an address higher that 4 gigabytes.
---  mmapping over 4G means we don't step on LuaJIT's toes, and this usually
---  works first time.
---  See `man mmap` for explanation of parameters.
---  @treturn pointer: the memory allocated.
-local function mmap_4G(
-  size,         -- integer: size to allocate in bytes
-  prot,         -- string: mmap's prot, as interpreted by syscall
-  flags,        -- string: mmap's flags, as interpreted by syscall
-  fd,           -- integer: file descriptor to map to
-  offset)       -- ?integer: offset into file to map
-  offset = offset or 0
-  local base = 4 * 1024 * 1024 * 1024
-  local step = 2^math.floor(math.log(tonumber(size)) / math.log(2))
-  local addr
-  while true do
-    addr = S.mmap(ffi.cast("void*", base), size, prot, flags, fd, offset)
-    if addr >= ffi.cast("void*", 4 * 1024 * 1024 * 1024) then break end
-    S.munmap(addr, size)
-    base = base + step
-  end
-  return addr
-end
-
-
-------------------------------------------------------------------------------
-
-local malloced_sizes = {}
-
-
---- "Map" some anonymous memory that's not mapped to a file.
---  This makes mmap behave a bit like malloc, except that we can persuade it
---  to give us memory above 4G, and malloc will be a very, very slow allocator
---  so only use it infrequently on big blocks of memory.
---  This is really just a hack to get us memory above 4G.  There's probably a
---  better solution.
---  @treturn pointer: the memory allocated.
-local function malloc(
-  size,         -- integer: number of bytes or `type`s to allocate.
-  type,         -- ?string: type to allocate
-  data)         -- ?pointer: data to copy to the mapped area.
-
-  if type then
-    size = size * ffi.sizeof(type)
-  end
-
-  local addr = assert(mmap_4G(size, "read, write", "anon, shared", fd))
-
-  malloced_sizes[tostring(ffi.cast("void*", addr))] = size
-
-  if data then
-    ffi.copy(addr, data, size)
-  end
-
-  if type then
-    return ffi.cast(type.."*", addr)
-  else
-    return addr
-  end
-end
-
-
---- Free memory mapped with mmapfile.malloc
---  Just munmaps the memory.
-local function free(
-  addr)         -- pointer: the mapped address to unmap.
-  local s = tostring(ffi.cast("void*", addr))
-  local size = assert(malloced_sizes[s], "no mmapped block at this address")
-  assert(S.munmap(addr, size))
-end
-
-
 --- Same as malloc, but set up a GC cleanup for the memory.
 --  @treturn pointer: the memory allocated
 local function gcmalloc(
   size,         -- integer: number of bytes or `type`s to allocate.
   type,         -- ?string: type to allocate
   data)         -- ?pointer: data to copy to the mapped area.
-  return ffi.gc(malloc(size, type, data), free)
+  return ffi.gc(platform_mmapfile.malloc(size, type, data), platform_mmapfile.free)
 end
 
 
 ------------------------------------------------------------------------------
 
-local open_fds = {}
-
-
---- Close a mapping between a file and an address.
---  `msync` the memory to its associated file, `munmap` the memory, and close
---  the file.
-local function close(
-  addr)         -- pointer: the mapped address to unmap.
-  local s = tostring(ffi.cast("void*", addr))
-  local fd = assert(open_fds[s], "no file open for this address")
-  open_fds[s] = nil
-
-  -- it seems that file descriptors get closed before final __gc calls in
-  -- some exit scenarios, so we don't worry too much if we can't
-  -- stat the fd
-  local st = fd:stat()
-  if st then
-    assert(S.msync(addr, st.size, "sync"))
-    assert(S.munmap(addr, st.size))
-    assert(fd:close())
-  end
-end
-
-
---- Allocate memory and create a new file mapped to it.
---  Use create to set aside an area of memory to write to a file.
---  If `type` is supplied then the pointer to the allocated memory is cast
---  to the correct type, and `size` is the number of `type`, not bytes,
---  to allocate.
---  If `data` is supplied, then the data at `data` is copied into the mapped
---  memory (and so written to the file).  It might make more sense just to
---  map the pointer `data` directly to the file, but that might require `data`
---  to be on a page boundary.
---  The file descriptor is saved in a table keyed to the address allocated
---  so that close can find the write fd to close when the memory is unmapped.
---  @treturn pointer: the memory allocated.
-local function create(
-  filename,     -- string: name of the file to create.
-  size,         -- integer: number of bytes or `type`s to allocate.
-  type,         -- ?string: type to allocate
-  data)         -- ?pointer: data to copy to the mapped area.
-  local fd = assert(S.open(filename, "RDWR, CREAT", "RUSR, WUSR, RGRP, ROTH"))
-
-  if type then
-    size = size * ffi.sizeof(type)
-  end
-
-  -- lseek gets stroppy if we try to seek the -1th byte, so let's just say
-  -- all files are at least one byte, even if theres's no actual data.
-  size = math.max(size, 1)
-  assert(fd:lseek(size-1, "set"))
-  assert(fd:write(ffi.new("char[1]", 0), 1))
-
-  local addr = assert(mmap_4G(size, "read, write", "file, shared", fd))
-
-  open_fds[tostring(ffi.cast("void*", addr))] = fd
-
-  if data then
-    ffi.copy(addr, data, size)
-  end
-
-  if type then
-    return ffi.cast(type.."*", addr)
-  else
-    return addr
-  end
-end
-
-
 --- Same as create, but set up a GC cleanup for the memory and file.
 --  @treturn pointer: the memory allocated
 local function gccreate(
   filename,     -- string: name of the file to create.
   size,         -- integer: number of bytes or `type`s to allocate.
   type,         -- ?string: type to allocate
   data)         -- ?pointer: data to copy to the mapped area.
-  return ffi.gc(create(filename, size, type, data), close)
-end
-
-
---- Map an existing file to an area of memory.
---  If `type` is present, the the pointer returned is cast to the `type*` and
---  the size returned is the number of `types`, not bytes.
---  @treturn pointer: the memory allocated.
---  @treturn int: size of the file, in bytes or `type`s.
-local function open(
-  filename,     -- string: name of the file to open.
-  type,         -- ?string: type to allocate (default void*)
-  mode,         -- ?string: open mode for the file "r" or "rw" (default "r")
-  size,         -- ?integer: size to map (in multiples of type).  Default
-                -- is file size
-  offset)       -- ?integer: offset into the file (default 0)
-  offset = offset or 0
-
-  mode = mode or "r"
-  local filemode, mapmode
-  if mode == "r" then
-    filemode = "rdonly"
-    mapmode = "read"
-  elseif mode == "rw" then
-    filemode = "rdwr"
-    mapmode = "read, write"
-  else
-    return nil, "unknown read/write mode"
-  end
-
-  local fd, message = S.open(filename, filemode, 0)
-  if not fd then
-    error(("mmapfile.open: Error opening %s: %s"):format(filename, message))
-  end
-
-  if not size then
-    local st = assert(fd:stat())
-    size = st.size
-  elseif type then
-    size = size * ffi.sizeof(type)
-  end
-
-  local addr = assert(mmap_4G(size, mapmode, "file, shared", fd, offset))
-
-  open_fds[tostring(ffi.cast("void*", addr))] = fd
-
-  if type then
-    return ffi.cast(type.."*", addr), math.floor(size / ffi.sizeof(type))
-  else
-    return addr, size
-  end
+  return ffi.gc(platform_mmapfile.create(filename, size, type, data), platform_mmapfile.close)
 end
 
 
@@ -259,23 +61,23 @@ local function gcopen(
   size,         -- ?integer: size to map (in multiples of type).  Default
                 -- is file size
   offset)       -- ?integer: offset into the file (default 0)
-  local addr, size = open(filename, type, mode, size, offset)
-  return ffi.gc(addr, close), size
+  local addr, actual_size = platform_mmapfile.open(filename, type, mode, size, offset)
+  return ffi.gc(addr, platform_mmapfile.close), actual_size
 end
 
 
 ------------------------------------------------------------------------------
 
 return
 {
-  free = free,
-  malloc = malloc,
-  gcmalloc = gcmalloc,
-  create = create,
-  gccreate = gccreate,
-  open = open,
-  gcopen = gcopen,
-  close = close,
+  free          = platform_mmapfile.free,
+  malloc        = platform_mmapfile.malloc,
+  gcmalloc      = gcmalloc,
+  create        = platform_mmapfile.create,
+  gccreate      = gccreate,
+  open          = platform_mmapfile.open,
+  gcopen        = gcopen,
+  close         = platform_mmapfile.close,
 }
 
 ------------------------------------------------------------------------------