Implement installing packages over Git

libs/calculate-deps.lua

@@ -16,81 +16,175 @@ limitations under the License.
 
 --]]
 
-local normalize = require('semver').normalize
 local gte = require('semver').gte
 local log = require('log').log
+local exec = require('exec')
 local queryDb = require('pkg').queryDb
 local colorize = require('pretty-print').colorize
+local queryGit = require('pkg').queryGit
+local normalize = require('semver').normalize
 
-return function (db, deps, newDeps)
-
-  local addDep, processDeps
-
-  function processDeps(dependencies)
-    if not dependencies then return end
-    for alias, dep in pairs(dependencies) do
-      local name, version = dep:match("^([^@]+)@?(.*)$")
-      if #version == 0 then
-        version = nil
-      end
-      if type(alias) == "number" then
-        alias = name:match("([^/]+)$")
-      end
-      if not name:find("/") then
-        error("Package names must include owner/name at a minimum")
-      end
-      if version then
-        local ok
-        ok, version = pcall(normalize, version)
-        if not ok then
-          error("Invalid dependency version: " .. dep)
-        end
-      end
-      addDep(alias, name, version)
+local processDeps
+local db, deps, configs
+
+local GIT_SCHEMES = {
+  "^https?://", -- over http/s
+  "^ssh://", -- over ssh
+  "^git://", -- over git
+  "^ftps?://", -- over ftp/s
+  "^[^:]+:", -- over ssh
+}
+
+local function isGit(dep)
+  for i = 1, #GIT_SCHEMES do
+    if dep:match(GIT_SCHEMES[i]) then
+      return true
     end
   end
+  return false
+end
+
+local function resolveDep(alias, dep)
+  -- match for author/name@version
+  local name, version = dep:match("^([^@]+)@?(.*)$")
+  -- resolve alias name, in case it's a number (an array index)
+  if type(alias) == "number" then
+    alias = name:match("([^/]+)$")
+  end
+
+  -- make sure owner is provided
+  if not name:find("/") then -- FIXME: this does match on `author/` or `/package`
+    error("Package names must include owner/name at a minimum")
+  end
 
-  function addDep(alias, name, version)
-    local meta = deps[alias]
-    if meta then
-      if name ~= meta.name then
-        local message = string.format("%s %s ~= %s",
-          alias, meta.name, name)
-        log("alias conflict", message, "failure")
-        return
-      end
-      if version then
-        if not gte(meta.version, version) then
-          local message = string.format("%s %s ~= %s",
-            alias, meta.version, version)
-          log("version conflict", message, "failure")
-          return
-        end
-      end
+  -- resolve version
+  if #version ~= 0 then
+    local ok
+    ok, version = pcall(normalize, version)
+    if not ok then
+      error("Invalid dependency version: " .. dep)
+    end
+  else
+    version = nil
+  end
+
+  -- check for already installed packages
+  local meta = deps[alias]
+  if meta then
+    -- is there an alias conflict?
+    if name ~= meta.name then
+      local message = string.format("%s %s ~= %s",
+        alias, meta.name, name)
+      log("alias conflict", message, "failure")
+    -- is there a version conflict?
+    elseif version and not gte(meta.version, version) then
+      local message = string.format("%s %s ~= %s",
+        alias, meta.version, version)
+      log("version conflict", message, "failure")
+    -- re-process package dependencies if everything is ok
     else
-      local author, pname = name:match("^([^/]+)/(.*)$")
-      local match, hash = db.match(author, pname, version)
-
-      if not match then
-        error("No such "
-          .. (version and "version" or "package") .. ": "
-          .. name
-          .. (version and '@' .. version or ''))
-      end
-      local kind
-      meta, kind, hash = assert(queryDb(db, hash))
-      meta.db = db
-      meta.hash = hash
-      meta.kind = kind
-      deps[alias] = meta
+      processDeps(meta.dependencies)
     end
+    return
+  end
 
-    processDeps(meta.dependencies)
+  -- extract author and package names from "author/package"
+  -- and match against the local db for the resources
+  -- if not available locally, and an upstream is set, match the upstream db
+  local author, pname = name:match("^([^/]+)/(.*)$")
+  local match, hash = db.match(author, pname, version)
+
+  -- no such package has been found locally nor upstream
+  if not match then
+    error("No such "
+      .. (version and "version" or "package") .. ": "
+      .. name
+      .. (version and '@' .. version or ''))
+  end
+
+  -- query package metadata, and mark it for installation
+  local kind
+  meta, kind, hash = assert(queryDb(db, hash))
+  meta.db = db
+  meta.hash = hash
+  meta.kind = kind
+  deps[alias] = meta
+
+  -- handle the dependencies of the module
+  processDeps(meta.dependencies)
+end
 
+-- TODO: implement git protocol over https, to be used in case `git` cli isn't available
+-- TODO: implement someway to specify a branch/tag when fetching
+-- TODO: implement handling git submodules, or shall we not?
+local function resolveGitDep(url)
+  -- fetch the repo tree, don't include any tags
+  log("fetching", colorize("highlight", url))
+  local _, stderr, code = exec("git", "--git-dir=" .. configs.database,
+    "fetch", "--no-tags", "--depth=1", url)
+
+  -- was the fetch successful?
+  if code ~= 0 then
+    if stderr:match("^ENOENT") then
+      error("Cannot find git. Please make sure git is installed and available.")
+    else
+      error((stderr:gsub("\n$", "")))
+    end
+  end
+
+  -- load the fetched module tree
+  local raw = db.storage.read("FETCH_HEAD")
+  local hash = raw:match("^(.-)\t\t.-\n$")
+  assert(hash and #hash ~= 0, "Unable to retrieve FETCH_HEAD\n" .. raw)
+  hash = db.loadAs("commit", hash).tree
+
+  -- query module's metadata, and match author/name
+  local meta, kind
+  meta, kind, hash = queryGit(db, hash)
+  assert(meta, "Unable to find a valid package")
+  local author, name = meta.name:match("^([^/]+)/(.*)$")
+
+  -- check for installed packages and their version
+  local oldMeta = deps[name]
+  if oldMeta and not gte(oldMeta.version, meta.version) then
+    local message = string.format("%s %s ~= %s",
+      name, oldMeta.version, meta.version)
+    log("version conflict", message, "failure")
+    return
+  end
+
+  -- create a ref/tags/author/name/version pointing to module's tree
+  db.write(author, name, meta.version, hash)
+
+  -- mark the dep for installation
+  meta.db = db
+  meta.hash = hash
+  meta.kind = kind
+  deps[name] = meta
+
+  -- handle the dependencies of the module
+  processDeps(meta.dependencies)
+end
+
+function processDeps(dependencies)
+  if not dependencies then return end
+  -- iterate through dependencies and resolve each entry
+  for alias, dep in pairs(dependencies) do
+    if isGit(dep) then
+      resolveGitDep(dep)
+    else
+      resolveDep(alias, dep)
+    end
   end
+end
 
+return function (core, depsMap, newDeps)
+  -- assign gitDb and depsMap as upvalues to be visible everywhere
+  -- then start processing newDeps
+  db, deps, configs = core.db, depsMap, core.config
   processDeps(newDeps)
 
+  -- collect all deps names and log them
   local names = {}
   for k in pairs(deps) do
     names[#names + 1] = k
@@ -103,6 +197,5 @@ return function (db, deps, newDeps)
       colorize("highlight", name), meta.path or meta.version))
   end
 
-
   return deps
 end

libs/core.lua

@@ -148,7 +148,7 @@ local function makeCore(config)
     end
     if meta.dependencies and kind == "tree" then
       local deps = {}
-      calculateDeps(core.db, deps, meta.dependencies)
+      calculateDeps(core, deps, meta.dependencies)
       meta.snapshot = installDeps(core.db, hash, deps, false)
       log("snapshot hash", meta.snapshot)
     end
@@ -424,7 +424,7 @@ local function makeCore(config)
     local kind, hash = assert(import(core.db, zfs, source, rules, true))
     assert(kind == "tree", "Only tree packages are supported for now")
     local deps = getInstalled(zfs, source)
-    calculateDeps(core.db, deps, meta.dependencies)
+    calculateDeps(core, deps, meta.dependencies)
     hash = installDeps(core.db, hash, deps, true)
     return makeZip(hash, target, luvi_source)
   end
@@ -491,7 +491,7 @@ local function makeCore(config)
     end
 
     local deps = {}
-    calculateDeps(core.db, deps, meta.dependencies)
+    calculateDeps(core, deps, meta.dependencies)
     local tagObj = db.loadAs("tag", hash)
     if tagObj.type ~= "tree" then
       error("Only tags pointing to trees are currently supported for make")
@@ -533,7 +533,7 @@ local function makeCore(config)
 
   function core.installList(path, newDeps)
     local deps = getInstalled(gfs, path)
-    calculateDeps(core.db, deps, newDeps)
+    calculateDeps(core, deps, newDeps)
     installDepsFs(core.db, gfs, path, deps, true)
     return deps
   end

libs/pkg.lua

@@ -22,9 +22,10 @@ Package Metadata Commands
 
 These commands work with packages metadata.
 
-pkg.query(fs, path) -> meta, path           - Query an on-disk path for package info.
-pkg.queryDb(db, path) -> meta, kind         - Query an in-db hash for package info.
-pky.normalize(meta) -> author, tag, version - Extract and normalize pkg info
+pkg.query(fs, path) -> meta, path            - Query an on-disk path for package info.
+pkg.queryDb(db, path) -> meta, kind, hash    - Query an in-db hash for package info.
+plg.queryGit(db, path) -> meta, kind, hash   - Query an in-db hash fetched with `git fetch` for package info.
+pky.normalize(meta) -> author, tag, version  - Extract and normalize pkg info
 ]]
 
 local isFile = require('git').modes.isFile
@@ -167,6 +168,46 @@ local function queryDb(db, hash)
   return meta, kind, hash
 end
 
+local function queryGit(db, hash)
+  local method = db.offlineLoadAny or db.load -- is rdb loaded?
+  local kind, value = method(hash)
+  if not kind then
+    error("Attempt to load the fetched tree")
+  elseif kind ~= "tree" then
+    error("Illegal kind: " .. kind)
+  end
+
+  local tree = listToMap(value)
+  local path = "tree:" .. hash
+  local entry = tree["package.lua"]
+  if entry then
+    path = path .. "/package.lua"
+  elseif tree["init.lua"] then
+    entry = tree["init.lua"]
+    path = path .. "/init.lua"
+  else
+    -- check if the tree only contains a single lua file, and treat it as a package.
+    -- since in most git hosting services you won't have blob-pointing tag,
+    -- this has to make some assumption (or otherwise not support it)
+    -- in this case, it makes the assumption that a single-file package's repo
+    -- only has a single lua file
+    for name, meta in pairs(tree) do
+      if name:sub(-4) == ".lua" and isFile(meta.mode) then
+        if entry then -- it contains more than a single lua file
+          return nil, "ENOENT: No package.lua or init.lua in tree:" .. hash
+        end
+        entry = tree[name]
+        path = "blob:" .. entry.hash
+        kind = "blob"
+        hash = entry.hash
+      end
+    end
+  end
+
+  local meta = evalModule(db.loadAs("blob", entry.hash), path)
+  return meta, kind, hash
+end
+
 local function normalize(meta)
   local author, tag = meta.name:match("^([^/]+)/(.*)$")
   return author, tag, semver.normalize(meta.version)
@@ -176,5 +217,6 @@ end
 return {
   query = query,
   queryDb = queryDb,
+  queryGit = queryGit,
   normalize = normalize,
 }

libs/rdb.lua

@@ -24,7 +24,8 @@ local httpCodec = require('http-codec')
 local websocketCodec = require('websocket-codec')
 local makeRemote = require('codec').makeRemote
 local deframe = require('git').deframe
-local decodeTag = require('git').decoders.tag
+local decoders = require('git').decoders
+local decodeTag = decoders.tag
 local verifySignature = require('verify-signature')
 
 local function connectRemote(url, timeout)
@@ -141,6 +142,12 @@ return function(db, url, timeout)
     return assert(db.offlineLoad(hash))
   end
 
+  function db.offlineLoadAny(hash)
+    local raw = assert(db.offlineLoad(hash), "no such hash")
+    local kind, value = deframe(raw)
+    return kind, decoders[kind](value)
+  end
+
   function db.fetch(list)
     local refs = {}
     repeat