Skip to content

Commit

Permalink
implement installing over git
Browse files Browse the repository at this point in the history
  • Loading branch information
@Bilal2453
Bilal2453 committed Feb 21, 2022
1 parent b1f1874 commit 77c8795
Show file tree
Hide file tree
Showing 3 changed files with 204 additions and 64 deletions.
211 changes: 151 additions & 60 deletions libs/calculate-deps.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,81 +16,173 @@ limitations under the License.
--]]

local normalize = require('semver').normalize
local gte = require('semver').gte
local log = require('log').log
local exec = require('exec')
local queryDb = require('pkg').queryDb
local colorize = require('pretty-print').colorize
local queryGit = require('pkg').queryGit
local normalize = require('semver').normalize

return function (db, deps, newDeps)

local addDep, processDeps

function processDeps(dependencies)
if not dependencies then return end
for alias, dep in pairs(dependencies) do
local name, version = dep:match("^([^@]+)@?(.*)$")
if #version == 0 then
version = nil
end
if type(alias) == "number" then
alias = name:match("([^/]+)$")
end
if not name:find("/") then
error("Package names must include owner/name at a minimum")
end
if version then
local ok
ok, version = pcall(normalize, version)
if not ok then
error("Invalid dependency version: " .. dep)
end
end
addDep(alias, name, version)
local gitSchemes = {
"^https?://", -- over http/s protocol
"^ssh://", -- over ssh protocol
"^git://", -- over git protocol
"^ftps?://", -- over ftp/s protocol
"^[^:]+:", -- over ssh protocol
}

local processDeps
local db, deps

local function isGit(dep)
for i = 1, #gitSchemes do
if dep:match(gitSchemes[i]) then
return true
end
end
return false
end

local function resolveDep(alias, dep)
-- match for author/name@version
local name, version = dep:match("^([^@]+)@?(.*)$")
-- resolve alias name, in case it's a number (an array index)
if type(alias) == "number" then
alias = name:match("([^/]+)$")
end

-- make sure owner is provided
if not name:find("/") then -- FIXME: this does match on `author/` or `/package`
error("Package names must include owner/name at a minimum")
end

function addDep(alias, name, version)
local meta = deps[alias]
if meta then
if name ~= meta.name then
local message = string.format("%s %s ~= %s",
alias, meta.name, name)
log("alias conflict", message, "failure")
return
end
if version then
if not gte(meta.version, version) then
local message = string.format("%s %s ~= %s",
alias, meta.version, version)
log("version conflict", message, "failure")
return
end
end
-- resolve version
if #version ~= 0 then
local ok
ok, version = pcall(normalize, version)
if not ok then
error("Invalid dependency version: " .. dep)
end
else
version = nil
end

-- check for already installed packages
local meta = deps[alias]
if meta then
-- is there an alias conflict?
if name ~= meta.name then
local message = string.format("%s %s ~= %s",
alias, meta.name, name)
log("alias conflict", message, "failure")
-- is there a version conflict?
elseif version and not gte(meta.version, version) then
local message = string.format("%s %s ~= %s",
alias, meta.version, version)
log("version conflict", message, "failure")
-- re-process package dependencies if everything is ok
else
local author, pname = name:match("^([^/]+)/(.*)$")
local match, hash = db.match(author, pname, version)

if not match then
error("No such "
.. (version and "version" or "package") .. ": "
.. name
.. (version and '@' .. version or ''))
end
local kind
meta, kind, hash = assert(queryDb(db, hash))
meta.db = db
meta.hash = hash
meta.kind = kind
deps[alias] = meta
processDeps(meta.dependencies)
end
return
end

processDeps(meta.dependencies)
-- extract author and package names from "author/package"
-- and match against the local db for the resources
-- if not available locally, and an upstream is set, match the upstream db
local author, pname = name:match("^([^/]+)/(.*)$")
local match, hash = db.match(author, pname, version)

-- no such package has been found locally nor upstream
if not match then
error("No such "
.. (version and "version" or "package") .. ": "
.. name
.. (version and '@' .. version or ''))
end

-- query package metadata, and mark it for installation
local kind
meta, kind, hash = assert(queryDb(db, hash))
meta.db = db
meta.hash = hash
meta.kind = kind
deps[alias] = meta

-- handle the dependencies of the module
processDeps(meta.dependencies)
end

-- TODO: implement git protocol over https, to be used in case `git` cli isn't available
-- TODO: implement someway to specify a branch/tag when fetching
-- TODO: implement handling git submodules
local function resolveGitDep(url)
-- fetch the repo tree, don't include any tags
log("fetching", colorize("highlight", url))
local _, stderr, code = exec("git", "fetch", "--no-tags", "--depth=1", "--recurse-submodules", url)

-- was the fetch successful?
if code ~= 0 then
if stderr:match("^ENOENT") then
error("Cannot find git. Please make sure git is installed and available.")
else
error(stderr:gsub("\n$", ""))
end
end

-- load the fetched module tree
local raw = db.storage.read("FETCH_HEAD")
local hash = raw:match("^(.-)\t\t.-\n$")
assert(hash and #hash ~= 0, "Attempt to retrive FETCH_HEAD")
hash = db.loadAs("commit", hash).tree

-- query module's metadata, and match author/name
local meta, kind
meta, kind, hash = assert(queryGit(db, hash))
local author, name = meta.name:match("^([^/]+)/(.*)$")

-- check for installed packages and their version
local oldMeta = deps[name]
if oldMeta and not gte(oldMeta.version, meta.version) then
local message = string.format("%s %s ~= %s",
name, oldMeta.version, meta.version)
log("version conflict", message, "failure")
return
end

-- create a ref/tags/author/name/version pointing to module's tree
db.write(author, name, meta.version, hash)

-- mark the dep for installation
meta.db = db
meta.hash = hash
meta.kind = kind
deps[name] = meta

-- handle the dependencies of the module
processDeps(meta.dependencies)
end

function processDeps(dependencies)
if not dependencies then return end
-- iterate through dependencies and resolve each entry
for alias, dep in pairs(dependencies) do
if isGit(dep) then
resolveGitDep(dep)
else
resolveDep(alias, dep)
end
end
end

return function (gitDb, depsMap, newDeps)
-- assign gitDb and depsMap to upvalue to be visible everywhere
-- then start processing newDeps
db, deps = gitDb, depsMap
processDeps(newDeps)

-- collect all deps names and log them
local names = {}
for k in pairs(deps) do
names[#names + 1] = k
Expand All @@ -103,6 +195,5 @@ return function (db, deps, newDeps)
colorize("highlight", name), meta.path or meta.version))
end


return deps
end
48 changes: 45 additions & 3 deletions libs/pkg.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ Package Metadata Commands
These commands work with packages metadata.
pkg.query(fs, path) -> meta, path - Query an on-disk path for package info.
pkg.queryDb(db, path) -> meta, kind - Query an in-db hash for package info.
pky.normalize(meta) -> author, tag, version - Extract and normalize pkg info
pkg.query(fs, path) -> meta, path - Query an on-disk path for package info.
pkg.queryDb(db, path) -> meta, kind, hash - Query an in-db hash for package info.
plg.queryGit(db, path) -> meta, kind, hash - Query an in-db hash fetched with `git fetch` for package info.
pky.normalize(meta) -> author, tag, version - Extract and normalize pkg info
]]

local isFile = require('git').modes.isFile
Expand Down Expand Up @@ -167,6 +168,46 @@ local function queryDb(db, hash)
return meta, kind, hash
end

local function queryGit(db, hash)
local method = db.offlineLoadAny or db.load -- is rdb loaded?
local kind, value = method(hash)
if not kind then
error("Attempt to load the fetched tree")
elseif kind ~= "tree" then
error("Illegal kind: " .. kind)
end

local tree = listToMap(value)
local path = "tree:" .. hash
local entry = tree["package.lua"]
if entry then
path = path .. "/package.lua"
elseif tree["init.lua"] then
entry = tree["init.lua"]
path = path .. "/init.lua"
else
-- check if the tree only contains a single lua file, and treat it as a package.
-- since in most git hosting services you won't have blob-pointing tag,
-- this has to make some assumption (or otherwise not support it)
-- in this case, it makes the assumption that a single-file package's repo
-- only has a single lua file
for name, meta in pairs(tree) do
if name:sub(-4) == ".lua" and isFile(meta.mode) then
if entry then -- it contains more than a single lua file
return nil, "ENOENT: No package.lua or init.lua in tree:" .. hash
end
entry = tree[name]
path = "blob:" .. entry.hash
kind = "blob"
hash = entry.hash
end
end
end

local meta = evalModule(db.loadAs("blob", entry.hash), path)
return meta, kind, hash
end

local function normalize(meta)
local author, tag = meta.name:match("^([^/]+)/(.*)$")
return author, tag, semver.normalize(meta.version)
Expand All @@ -176,5 +217,6 @@ end
return {
query = query,
queryDb = queryDb,
queryGit = queryGit,
normalize = normalize,
}
9 changes: 8 additions & 1 deletion libs/rdb.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ local httpCodec = require('http-codec')
local websocketCodec = require('websocket-codec')
local makeRemote = require('codec').makeRemote
local deframe = require('git').deframe
local decodeTag = require('git').decoders.tag
local decoders = require('git').decoders
local decodeTag = decoders.tag
local verifySignature = require('verify-signature')

local function connectRemote(url, timeout)
Expand Down Expand Up @@ -141,6 +142,12 @@ return function(db, url, timeout)
return assert(db.offlineLoad(hash))
end

function db.offlineLoadAny(hash)
local raw = assert(db.offlineLoad(hash), "no such hash")
local kind, value = deframe(raw)
return kind, decoders[kind](value)
end

function db.fetch(list)
local refs = {}
repeat
Expand Down

0 comments on commit 77c8795

Please sign in to comment.