From a61dfde2036a0c5e4303ae81f4841ae9b405dc63 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 28 Mar 2024 11:17:20 +0000 Subject: [PATCH 1/4] Removes noise from SAT implementation disabled tests will be added in a future PR --- nimble.nimble | 2 +- src/nimble.nim | 114 ++++++----- src/nimblepkg/download.nim | 68 ++++++- src/nimblepkg/nimblesat.nim | 364 ++++++++++++++++++++++++++++++++++++ src/nimblepkg/options.nim | 5 + src/nimblepkg/version.nim | 2 +- tests/tester.nim | 2 +- tests/tsat.nim | 239 +++++++++++++++++++++++ 8 files changed, 731 insertions(+), 65 deletions(-) create mode 100644 src/nimblepkg/nimblesat.nim create mode 100644 tests/tsat.nim diff --git a/nimble.nimble b/nimble.nimble index 2dc487aa..cee78ab7 100644 --- a/nimble.nimble +++ b/nimble.nimble @@ -11,7 +11,7 @@ installExt = @["nim"] # Dependencies -requires "nim >= 0.13.0" +requires "nim >= 0.13.0", "sat" when defined(nimdistros): import distros diff --git a/src/nimble.nim b/src/nimble.nim index 2f64a262..44d9db5e 100644 --- a/src/nimble.nim +++ b/src/nimble.nim @@ -1,6 +1,8 @@ # Copyright (C) Dominik Picheta. All rights reserved. # BSD License. Look at license.txt for more info. +import system except TResult + import os, tables, strtabs, json, algorithm, sets, uri, sugar, sequtils, osproc, strformat @@ -8,9 +10,9 @@ import std/options as std_opt import strutils except toLower from unicode import toLower - +import sat/sat import nimblepkg/packageinfotypes, nimblepkg/packageinfo, nimblepkg/version, - nimblepkg/tools, nimblepkg/download, nimblepkg/config, nimblepkg/common, + nimblepkg/tools, nimblepkg/download, nimblepkg/common, nimblepkg/publish, nimblepkg/options, nimblepkg/packageparser, nimblepkg/cli, nimblepkg/packageinstaller, nimblepkg/reversedeps, nimblepkg/nimscriptexecutor, nimblepkg/init, nimblepkg/vcstools, @@ -18,7 +20,7 @@ import nimblepkg/packageinfotypes, nimblepkg/packageinfo, nimblepkg/version, nimblepkg/nimscriptwrapper, nimblepkg/developfile, nimblepkg/paths, nimblepkg/nimbledatafile, nimblepkg/packagemetadatafile, nimblepkg/displaymessages, nimblepkg/sha1hashes, nimblepkg/syncfile, - nimblepkg/deps + nimblepkg/deps, nimblepkg/nimblesat const nimblePathsFileName* = "nimble.paths" @@ -28,34 +30,6 @@ const nimblePathsEnv = "__NIMBLE_PATHS" separator = when defined(windows): ";" else: ":" -proc refresh(options: Options) = - ## Downloads the package list from the specified URL. - ## - ## If the download is not successful, an exception is raised. - if options.offline: - raise nimbleError("Cannot refresh package list in offline mode.") - - let parameter = - if options.action.typ == actionRefresh: - options.action.optionalURL - else: - "" - - if parameter.len > 0: - if parameter.isUrl: - let cmdLine = PackageList(name: "commandline", urls: @[parameter]) - fetchList(cmdLine, options) - else: - if parameter notin options.config.packageLists: - let msg = "Package list with the specified name not found." - raise nimbleError(msg) - - fetchList(options.config.packageLists[parameter], options) - else: - # Try each package list in config - for name, list in options.config.packageLists: - fetchList(list, options) - proc initPkgList(pkgInfo: PackageInfo, options: Options): seq[PackageInfo] = let installedPkgs = getInstalledPkgsMin(options.getPkgsDir(), options) @@ -79,6 +53,50 @@ proc checkSatisfied(options: Options, dependencies: seq[PackageInfo]) = [pkgInfo.basicInfo.name, $currentVer, $pkgsInPath[pkgInfo.basicInfo.name]]) pkgsInPath[pkgInfo.basicInfo.name] = currentVer +proc processFreeDependenciesSAT(rootPkgInfo: PackageInfo, pkgList: seq[PackageInfo], options: Options): HashSet[PackageInfo] = + result = solveLocalPackages(rootPkgInfo, pkgList) + if result.len > 0: return result + + var reverseDependencies: seq[PackageBasicInfo] = @[] + var pkgsToInstall: seq[(string, Version)] = @[] + var output = "" + result = solvePackages(rootPkgInfo, pkgList, pkgsToInstall, options, output) + if pkgsToInstall.len > 0: + for pkg in pkgsToInstall: + let dep = (name: pkg[0], ver: pkg[1].toVersionRange) + let resolvedDep = dep.resolveAlias(options) + display("Installing", $resolvedDep, priority = HighPriority) + let toInstall = @[(resolvedDep.name, resolvedDep.ver)] + #TODO install here will download the package again. We could use the already downloaded package + #from the cache + let (packages, _) = install(toInstall, options, + doPrompt = false, first = false, fromLockFile = false, preferredPackages = @[]) + + for pkg in packages: + if result.contains pkg: + # If the result already contains the newly tried to install package + # we had to merge its special versions set into the set of the old + # one. + result[pkg].metaData.specialVersions.incl( + pkg.metaData.specialVersions) + else: + result.incl pkg + + if not pkg.isLink: + reverseDependencies.add(pkg.basicInfo) + if result.len > 0: + # We add the reverse deps to the JSON file here because we don't want + # them added if the above errorenous condition occurs + # (unsatisfiable dependendencies). + # N.B. NimbleData is saved in installFromDir. + for i in reverseDependencies: + addRevDep(options.nimbleData, i, rootPkgInfo) + return result + else: + display("Error", output, Error, priority = HighPriority) + raise nimbleError("Unsatisfiable dependencies") + + proc processFreeDependencies(pkgInfo: PackageInfo, requirements: seq[PkgTuple], options: Options, @@ -92,7 +110,12 @@ proc processFreeDependencies(pkgInfo: PackageInfo, "processFreeDependencies needs pkgInfo.requires" var pkgList {.global.}: seq[PackageInfo] - once: pkgList = initPkgList(pkgInfo, options) + + once: + pkgList = initPkgList(pkgInfo, options) + if options.useSatSolver: + return processFreeDependenciesSAT(pkgInfo, pkgList, options) + display("Verifying", "dependencies for $1@$2" % [pkgInfo.basicInfo.name, $pkgInfo.basicInfo.version], priority = HighPriority) @@ -666,33 +689,6 @@ proc processLockedDependencies(pkgInfo: PackageInfo, options: Options): return res.toHashSet -proc getDownloadInfo*(pv: PkgTuple, options: Options, - doPrompt: bool, ignorePackageCache = false): (DownloadMethod, string, - Table[string, string]) = - if pv.name.isURL: - let (url, metadata) = getUrlData(pv.name) - return (checkUrlType(url), url, metadata) - else: - var pkg = initPackage() - if getPackage(pv.name, options, pkg, ignorePackageCache): - let (url, metadata) = getUrlData(pkg.url) - return (pkg.downloadMethod, url, metadata) - else: - # If package is not found give the user a chance to refresh - # package.json - if doPrompt and not options.offline and - options.prompt(pv.name & " not found in any local packages.json, " & - "check internet for updated packages?"): - refresh(options) - - # Once we've refreshed, try again, but don't prompt if not found - # (as we've already refreshed and a failure means it really - # isn't there) - # Also ignore the package cache so the old info isn't used - return getDownloadInfo(pv, options, false, true) - else: - raise nimbleError(pkgNotFoundMsg(pv)) - proc compileNim(realDir: string) = let command = when defined(windows): "build_all.bat" else: "./build_all.sh" cd realDir: diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index 5318a8f0..c4252d3d 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -7,7 +7,7 @@ import parseutils, os, osproc, strutils, tables, pegs, uri, strformat, from algorithm import SortOrder, sorted import packageinfotypes, packageparser, version, tools, common, options, cli, - sha1hashes, vcstools + sha1hashes, vcstools, displaymessages, packageinfo, config type DownloadPkgResult* = tuple @@ -461,13 +461,20 @@ proc downloadPkg*(url: string, verRange: VersionRange, if options.offline: raise nimbleError("Cannot download in offline mode.") - let downloadDir = if downloadPath == "": - (getNimbleTempDir() / getDownloadDirName(url, verRange, vcsRevision)) + let dir = if options.pkgCachePath != "": + options.pkgCachePath + else: + getNimbleTempDir() + (dir / getDownloadDirName(url, verRange, vcsRevision)) else: downloadPath + if options.pkgCachePath != "" and dirExists(downloadDir): + #TODO test integrity of the package + return (dir: downloadDir, version: newVersion getSimpleString(verRange), vcsRevision: notSetSha1Hash) + createDir(downloadDir) var modUrl = if url.startsWith("git://") and options.config.cloneUsingHttps: @@ -544,6 +551,61 @@ proc getDevelopDownloadDir*(url, subdir: string, options: Options): string = else: getCurrentDir() / options.action.path / downloadDirName +proc refresh*(options: Options) = + ## Downloads the package list from the specified URL. + ## + ## If the download is not successful, an exception is raised. + if options.offline: + raise nimbleError("Cannot refresh package list in offline mode.") + + let parameter = + if options.action.typ == actionRefresh: + options.action.optionalURL + else: + "" + + if parameter.len > 0: + if parameter.isUrl: + let cmdLine = PackageList(name: "commandline", urls: @[parameter]) + fetchList(cmdLine, options) + else: + if parameter notin options.config.packageLists: + let msg = "Package list with the specified name not found." + raise nimbleError(msg) + + fetchList(options.config.packageLists[parameter], options) + else: + # Try each package list in config + for name, list in options.config.packageLists: + fetchList(list, options) + +proc getDownloadInfo*(pv: PkgTuple, options: Options, + doPrompt: bool, ignorePackageCache = false): (DownloadMethod, string, + Table[string, string]) = + if pv.name.isURL: + let (url, metadata) = getUrlData(pv.name) + return (checkUrlType(url), url, metadata) + else: + var pkg = initPackage() + if getPackage(pv.name, options, pkg, ignorePackageCache): + let (url, metadata) = getUrlData(pkg.url) + return (pkg.downloadMethod, url, metadata) + else: + # If package is not found give the user a chance to refresh + # package.json + if doPrompt and not options.offline and + options.prompt(pv.name & " not found in any local packages.json, " & + "check internet for updated packages?"): + refresh(options) + + # Once we've refreshed, try again, but don't prompt if not found + # (as we've already refreshed and a failure means it really + # isn't there) + # Also ignore the package cache so the old info isn't used + return getDownloadInfo(pv, options, false, true) + else: + raise nimbleError(pkgNotFoundMsg(pv)) + when isMainModule: import unittest diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim new file mode 100644 index 00000000..02630c62 --- /dev/null +++ b/src/nimblepkg/nimblesat.nim @@ -0,0 +1,364 @@ +import sat/[sat, satvars] +import version, packageinfotypes, download, packageinfo, packageparser, options, + sha1hashes + +import std/[tables, sequtils, algorithm, sets, strutils, options, strformat] + + +type + SatVarInfo* = object # attached information for a SAT variable + pkg*: string + version*: Version + index*: int + + Form* = object + f*: Formular + mapping*: Table[VarId, SatVarInfo] + idgen*: int32 + + PackageMinimalInfo* = object + name*: string + version*: Version + requires*: seq[PkgTuple] + isRoot*: bool + + PackageVersions* = object + pkgName*: string + versions*: seq[PackageMinimalInfo] + + Requirements* = object + deps*: seq[PkgTuple] #@[(name, versRange)] + version*: Version + nimVersion*: Version + v*: VarId + err*: string + + DependencyVersion* = object # Represents a specific version of a project. + version*: Version + req*: int # index into graph.reqs so that it can be shared between versions + v*: VarId + # req: Requirements + + Dependency* = object + pkgName*: string + versions*: seq[DependencyVersion] + active*: bool + activeVersion*: int + isRoot*: bool + + DepGraph* = object + nodes*: seq[Dependency] + reqs*: seq[Requirements] + packageToDependency*: Table[string, int] #package.name -> index into nodes + # reqsByDeps: Table[Requirements, int] + GetPackageMinimal* = proc (pv: PkgTuple, options: Options): Option[PackageMinimalInfo] + +proc getMinimalInfo*(pkg: PackageInfo): PackageMinimalInfo = + result.name = pkg.basicInfo.name + result.version = pkg.basicInfo.version + result.requires = pkg.requires + +proc hasVersion*(packageVersions: PackageVersions, pv: PkgTuple): bool = + for pkg in packageVersions.versions: + if pkg.name == pv.name and pkg.version.withinRange(pv.ver): + return true + false + +proc hasVersion*(packagesVersions: Table[string, PackageVersions], pv: PkgTuple): bool = + if pv.name in packagesVersions: + return packagesVersions[pv.name].hasVersion(pv) + false + +proc hasVersion*(packagesVersions: Table[string, PackageVersions], name: string, ver: Version): bool = + if name in packagesVersions: + for pkg in packagesVersions[name].versions: + if pkg.version == ver: + return true + false + +proc getNimVersion*(pvs: seq[PkgTuple]): Version = + proc getVersion(ver: VersionRange): Version = + case ver.kind: + of verLater, verEarlier, verEqLater, verEqEarlier, verEq: + ver.ver + of verSpecial: + ver.spe + of verIntersect, verTilde, verCaret: + getVersion(ver.verILeft) + of verAny: + newVersion "0.0.0" + + result = newVersion("0.0.0") + for pv in pvs: + if pv.name == "nim": + result = getVersion(pv.ver) + +proc findDependencyForDep(g: DepGraph; dep: string): int {.inline.} = + assert g.packageToDependency.hasKey(dep), dep & " not found" + result = g.packageToDependency.getOrDefault(dep) + +proc createRequirements(pkg: PackageMinimalInfo): Requirements = + result.deps = pkg.requires.filterIt(it.name != "nim") + result.version = pkg.version + result.nimVersion = pkg.requires.getNimVersion() + +proc cmp(a,b: DependencyVersion): int = + if a.version < b.version: return -1 + elif a.version == b.version: return 0 + else: return 1 + +proc getRequirementFromGraph(g: var DepGraph, pkg: PackageMinimalInfo): int = + var temp = createRequirements(pkg) + for i in countup(0, g.reqs.len-1): + if g.reqs[i] == temp: return i + g.reqs.add temp + g.reqs.len-1 + +proc toDependencyVersion(g: var DepGraph, pkg: PackageMinimalInfo): DependencyVersion = + result.version = pkg.version + result.req = getRequirementFromGraph(g, pkg) + +proc toDependency(g: var DepGraph, pkg: PackageVersions): Dependency = + result.pkgName = pkg.pkgName + result.versions = pkg.versions.mapIt(toDependencyVersion(g, it)) + assert pkg.versions.len > 0, "Package must have at least one version" + result.isRoot = pkg.versions[0].isRoot + +proc toDepGraph*(versions: Table[string, PackageVersions]): DepGraph = + var root: PackageVersions + for pv in versions.values: + if pv.versions[0].isRoot: + root = pv + else: + result.nodes.add toDependency(result, pv) + assert root.pkgName != "", "No root package found" + result.nodes.insert(toDependency(result, root), 0) + # Fill the other field and I should be good to go? + for i in countup(0, result.nodes.len-1): + result.packageToDependency[result.nodes[i].pkgName] = i + +proc toFormular*(g: var DepGraph): Form = +# Key idea: use a SAT variable for every `Requirements` object, which are +# shared. + result = Form() + var b = Builder() + b.openOpr(AndForm) + # Assign a variable for each package version + for p in mitems(g.nodes): + if p.versions.len == 0: continue + p.versions.sort(cmp) + + for ver in mitems p.versions: + ver.v = VarId(result.idgen) + result.mapping[ver.v] = SatVarInfo(pkg: p.pkgName, version: ver.version, index: result.idgen) + inc result.idgen + + # Encode the rule: for root packages, exactly one of its versions must be true + if p.isRoot: + b.openOpr(ExactlyOneOfForm) + for ver in mitems p.versions: + b.add(ver.v) + b.closeOpr() + else: + # For non-root packages, either one version is selected or none + b.openOpr(ZeroOrOneOfForm) + for ver in mitems p.versions: + b.add(ver.v) + b.closeOpr() + + # Model dependencies and their version constraints + for p in mitems(g.nodes): + for ver in p.versions.mitems: + let eqVar = VarId(result.idgen) + # Mark the beginning position for a potential reset + let beforeDeps = b.getPatchPos() + + inc result.idgen + var hasDeps = false + + for dep, q in items g.reqs[ver.req].deps: + let av = g.nodes[findDependencyForDep(g, dep)] + if av.versions.len == 0: continue + + hasDeps = true + b.openOpr(ExactlyOneOfForm) # Dependency must satisfy at least one of the version constraints + + for avVer in av.versions: + if avVer.version.withinRange(q): + b.add(avVer.v) # This version of the dependency satisfies the constraint + + b.closeOpr() + + # If the package version is chosen and it has dependencies, enforce the dependencies' constraints + if hasDeps: + b.openOpr(OrForm) + b.addNegated(ver.v) # If this package version is not chosen, skip the dependencies constraint + b.add(eqVar) # Else, ensure the dependencies' constraints are met + b.closeOpr() + + # If no dependencies were added, reset to beforeDeps to avoid empty or invalid operations + if not hasDeps: + b.resetToPatchPos(beforeDeps) + + b.closeOpr() # Close the main AndForm + result.f = toForm(b) # Convert the builder to a formula + +proc toString(x: SatVarInfo): string = + "(" & x.pkg & ", " & $x.version & ")" + +proc debugFormular*(g: var DepGraph; f: Form; s: Solution) = + echo "FORM: ", f.f + #for n in g.nodes: + # echo "v", n.v.int, " ", n.pkg.url + for k, v in pairs(f.mapping): + echo "v", k.int, ": ", v + let m = maxVariable(f.f) + for i in 0 ..< m: + if s.isTrue(VarId(i)): + echo "v", i, ": T" + else: + echo "v", i, ": F" + +proc getNodeByReqIdx(g: var DepGraph, reqIdx: int): Option[Dependency] = + for n in g.nodes: + if n.versions.anyIt(it.req == reqIdx): + return some n + none(Dependency) + +proc generateUnsatisfiableMessage(g: var DepGraph, f: Form, s: Solution): string = + var conflicts: seq[string] = @[] + for reqIdx, req in g.reqs: + if not s.isTrue(req.v): # Check if the requirement's corresponding variable was not satisfied + for dep in req.deps: + var dep = dep + let depNodeIdx = findDependencyForDep(g, dep.name) + let depVersions = g.nodes[depNodeIdx].versions + let satisfiableVersions = + depVersions.filterIt(it.version.withinRange(dep.ver) and s.isTrue(it.v)) + + if satisfiableVersions.len == 0: + # No version of this dependency could satisfy the requirement + # Find which package/version had this requirement + let reqNode = g.getNodeByReqIdx(reqIdx) + if reqNode.isSome: + let pkgName = reqNode.get.pkgName + conflicts.add(&"Requirement '{dep.name} {dep.ver}' required by '{pkgName} {req.version}' could not be satisfied.") + + if conflicts.len == 0: + return "Dependency resolution failed due to unsatisfiable dependencies, but specific conflicts could not be determined." + else: + return "Dependency resolution failed due to the following conflicts:\n" & conflicts.join("\n") + +#It may be better to just use result here +proc solve*(g: var DepGraph; f: Form, packages: var Table[string, Version], output: var string): bool = + let m = f.idgen + var s = createSolution(m) + if satisfiable(f.f, s): + for n in mitems g.nodes: + if n.isRoot: n.active = true + for i in 0 ..< m: + if s.isTrue(VarId(i)) and f.mapping.hasKey(VarId i): + let m = f.mapping[VarId i] + let idx = findDependencyForDep(g, m.pkg) + g.nodes[idx].active = true + g.nodes[idx].activeVersion = m.index + + for n in items g.nodes: + for v in items(n.versions): + let item = f.mapping[v.v] + if s.isTrue(v.v): + packages[item.pkg] = item.version + output.add &"item.pkg [x] {toString item} \n" + else: + output.add &"item.pkg [ ] {toString item} \n" + true + else: + #TODO we could make a permuted version of the requires for the root package and try again + output = generateUnsatisfiableMessage(g, f, s) + false + +proc getSolvedPackages*(pkgVersionTable: Table[string, PackageVersions], output: var string): Table[string, Version] = + var graph = pkgVersionTable.toDepGraph() + #Make sure all references are in the graph before calling toFormular + for p in graph.nodes: + for ver in p.versions.items: + for dep, q in items graph.reqs[ver.req].deps: + if dep notin graph.packageToDependency: + output.add &"Dependency {dep} not found in the graph \n" + return initTable[string, Version]() + + let form = toFormular(graph) + var packages = initTable[string, Version]() + discard solve(graph, form, packages, output) + packages + +proc downloadPkInfoForPv*(pv: PkgTuple, options: Options): PackageInfo = + let (meth, url, metadata) = + getDownloadInfo(pv, options, doPrompt = true) + let subdir = metadata.getOrDefault("subdir") + let res = + downloadPkg(url, pv.ver, meth, subdir, options, + "", vcsRevision = notSetSha1Hash) + return getPkgInfo(res.dir, options) + +proc downloadMinimalPackage*(pv: PkgTuple, options: Options): Option[PackageMinimalInfo] = + if pv.name == "": return none(PackageMinimalInfo) + # echo "Downloading ", pv.name, " ", pv.ver + let pkgInfo = downloadPkInfoForPv(pv, options) + some pkgInfo.getMinimalInfo() + +proc fillPackageTableFromPreferred*(packages: var Table[string, PackageVersions], preferredPackages: seq[PackageMinimalInfo]) = + for pkg in preferredPackages: + if not hasVersion(packages, pkg.name, pkg.version): + if not packages.hasKey(pkg.name): + packages[pkg.name] = PackageVersions(pkgName: pkg.name, versions: @[pkg]) + else: + packages[pkg.name].versions.add pkg + +proc getInstalledMinimalPackages*(options: Options): seq[PackageMinimalInfo] = + getInstalledPkgsMin(options.getPkgsDir(), options).mapIt(it.getMinimalInfo()) + +proc collectAllVersions*(versions: var Table[string, PackageVersions], package: PackageMinimalInfo, options: Options, getMinimalPackage: GetPackageMinimal) = + ### Collects all the versions of a package and its dependencies and stores them in the versions table + ### A getMinimalPackage function is passed to get the package + for pv in package.requires: + # echo "Collecting versions for ", pv.name, " and Version: ", $pv.ver, " via ", package.name + var pv = pv + if not hasVersion(versions, pv): # Not found, meaning this package-version needs to be explored + var pkgMin = getMinimalPackage(pv, options).get() #TODO elegantly fail here + if pv.ver.kind == verSpecial: + pkgMin.version = newVersion $pv.ver + if not versions.hasKey(pv.name): + versions[pv.name] = PackageVersions(pkgName: pv.name, versions: @[pkgMin]) + else: + versions[pv.name].versions.addUnique pkgMin + collectAllVersions(versions, pkgMin, options, getMinimalPackage) + +proc solveLocalPackages*(rootPkgInfo: PackageInfo, pkgList: seq[PackageInfo]): HashSet[PackageInfo] = + var root = rootPkgInfo.getMinimalInfo() + root.isRoot = true + var pkgVersionTable = initTable[string, PackageVersions]() + pkgVersionTable[root.name] = PackageVersions(pkgName: root.name, versions: @[root]) + fillPackageTableFromPreferred(pkgVersionTable, pkgList.map(getMinimalInfo)) + var output = "" + var solvedPkgs = pkgVersionTable.getSolvedPackages(output) + for pkg, ver in solvedPkgs: + for pkgInfo in pkgList: + if pkgInfo.basicInfo.name == pkg and pkgInfo.basicInfo.version == ver: + result.incl pkgInfo + +proc solvePackages*(rootPkg: PackageInfo, pkgList: seq[PackageInfo], pkgsToInstall: var seq[(string, Version)], options: Options, output: var string): HashSet[PackageInfo] = + var root = rootPkg.getMinimalInfo() + root.isRoot = true + var pkgVersionTable = initTable[string, PackageVersions]() + pkgVersionTable[root.name] = PackageVersions(pkgName: root.name, versions: @[root]) + collectAllVersions(pkgVersionTable, root, options, downloadMinimalPackage) + var solvedPkgs = pkgVersionTable.getSolvedPackages(output) + var pkgsToInstall: seq[(string, Version)] = @[] + for solvedPkg, ver in solvedPkgs: + if solvedPkg == root.name: continue + for pkgInfo in pkgList: + if pkgInfo.basicInfo.name == solvedPkg: # and pkgInfo.basicInfo.version.withinRange(ver): + result.incl pkgInfo + else: + pkgsToInstall.addUnique((solvedPkg, ver)) \ No newline at end of file diff --git a/src/nimblepkg/options.nim b/src/nimblepkg/options.nim index 1a688e17..03b89940 100644 --- a/src/nimblepkg/options.nim +++ b/src/nimblepkg/options.nim @@ -55,6 +55,8 @@ type # For which package in the dependency tree the command should be executed. # If not provided by default it applies to the current directory package. # For now, it is used only by the run action and it is ignored by others. + pkgCachePath*: string # Cache used to store package downloads + useSatSolver*: bool ActionType* = enum actionNil, actionRefresh, actionInit, actionDump, actionPublish, actionUpgrade @@ -435,6 +437,8 @@ proc setNimbleDir*(options: var Options) = let pkgsDir = options.getPkgsDir() if not dirExists(pkgsDir): createDir(pkgsDir) + if options.useSatSolver: + options.pkgCachePath = options.getNimbleDir() / "pkgcache" proc parseCommand*(key: string, result: var Options) = result.action = Action(typ: parseActionType(key)) @@ -546,6 +550,7 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) = result.developFile = val.normalizedPath else: raise nimbleError(multipleDevelopFileOptionsGivenMsg) + of "sat": result.useSatSolver = true else: isGlobalFlag = false var wasFlagHandled = true diff --git a/src/nimblepkg/version.nim b/src/nimblepkg/version.nim index 956b5071..921a3c51 100644 --- a/src/nimblepkg/version.nim +++ b/src/nimblepkg/version.nim @@ -29,7 +29,7 @@ type of verSpecial: spe*: Version of verIntersect, verTilde, verCaret: - verILeft, verIRight: VersionRange + verILeft*, verIRight*: VersionRange of verAny: nil diff --git a/tests/tester.nim b/tests/tester.nim index 69cbde84..6ca975cb 100644 --- a/tests/tester.nim +++ b/tests/tester.nim @@ -28,7 +28,7 @@ import ttestcommand import ttwobinaryversions import tuninstall import ttaskdeps - +import tsat # nonim tests are very slow and (often) break the CI. # import tnonim diff --git a/tests/tsat.nim b/tests/tsat.nim new file mode 100644 index 00000000..a4d1605a --- /dev/null +++ b/tests/tsat.nim @@ -0,0 +1,239 @@ +{.used.} +import unittest, os +import testscommon +# from nimblepkg/common import cd Used in the commented tests +import std/[tables, sequtils, json, jsonutils, strutils] +import nimblepkg/[version, nimblesat, options, config] + + +proc initFromJson*(dst: var PkgTuple, jsonNode: JsonNode, jsonPath: var string) = + dst = parseRequires(jsonNode.str) + +proc toJsonHook*(src: PkgTuple): JsonNode = + let ver = if src.ver.kind == verAny: "" else: $src.ver + case src.ver.kind + of verAny: newJString(src.name) + of verSpecial: newJString(src.name & ver) + else: + newJString(src.name & " " & ver) + +#Test utils: +proc downloadAndStorePackageVersionTableFor(pkgName: string, options: Options) = + #Downloads all the dependencies for a given package and store the minimal version of the deps in a json file. + var fileName = pkgName + if pkgName.startsWith("https://"): + let pkgUrl = pkgName + fileName = pkgUrl.split("/")[^1].split(".")[0] + + let path = "packageMinimal" / fileName & ".json" + if fileExists(path): + return + let pv: PkgTuple = (pkgName, VersionRange(kind: verAny)) + var pkgInfo = downloadPkInfoForPv(pv, options) + var root = pkgInfo.getMinimalInfo() + root.isRoot = true + var pkgVersionTable = initTable[string, PackageVersions]() + collectAllVersions(pkgVersionTable, root, options, downloadMinimalPackage) + pkgVersionTable[pkgName] = PackageVersions(pkgName: pkgName, versions: @[root]) + let json = pkgVersionTable.toJson() + writeFile(path, json.pretty()) + +proc downloadAllPackages() {.used.} = + var options = initOptions() + options.nimBin = "nim" + # options.config.packageLists["uing"] = PackageList(name: pkgName, urls: @[pkgUrl]) + options.config.packageLists["official"] = PackageList(name: "Official", urls: @[ + "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json", + "https://nim-lang.org/nimble/packages.json" + ]) + + # let packages = getPackageList(options).mapIt(it.name) + let importantPackages = [ + "alea", "argparse", "arraymancer", "ast_pattern_matching", "asyncftpclient", "asyncthreadpool", "awk", "bigints", "binaryheap", "BipBuffer", "blscurve", + "bncurve", "brainfuck", "bump", "c2nim", "cascade", "cello", "checksums", "chroma", "chronicles", "chronos", "cligen", "combparser", "compactdict", + "https://github.com/alehander92/comprehension", "cowstrings", "criterion", "datamancer", "dashing", "delaunay", "docopt", "drchaos", "https://github.com/jackmott/easygl", "elvis", "fidget", "fragments", "fusion", "gara", "glob", "ggplotnim", + "https://github.com/disruptek/gittyup", "gnuplot", "https://github.com/disruptek/gram", "hts", "httpauth", "illwill", "inim", "itertools", "iterutils", "jstin", "karax", "https://github.com/jblindsay/kdtree", "loopfusion", "lockfreequeues", "macroutils", "manu", "markdown", + "measuremancer", "memo", "msgpack4nim", "nake", "https://github.com/nim-lang/neo", "https://github.com/nim-lang/NESM", "netty", "nico", "nicy", "nigui", "nimcrypto", "NimData", "nimes", "nimfp", "nimgame2", "nimgen", "nimib", "nimlsp", "nimly", + "nimongo", "https://github.com/disruptek/nimph", "nimPNG", "nimpy", "nimquery", "nimsl", "nimsvg", "https://github.com/nim-lang/nimterop", "nimwc", "nimx", "https://github.com/zedeus/nitter", "norm", "npeg", "numericalnim", "optionsutils", "ormin", "parsetoml", "patty", "pixie", + "plotly", "pnm", "polypbren", "prologue", "protobuf", "pylib", "rbtree", "react", "regex", "results", "RollingHash", "rosencrantz", "sdl1", "sdl2_nim", "sigv4", "sim", "smtp", "https://github.com/genotrance/snip", "ssostrings", + "stew", "stint", "strslice", "strunicode", "supersnappy", "synthesis", "taskpools", "telebot", "tempdir", "templates", "https://krux02@bitbucket.org/krux02/tensordslnim.git", "terminaltables", "termstyle", "timeit", "timezones", "tiny_sqlite", + "unicodedb", "unicodeplus", "https://github.com/alaviss/union", "unpack", "weave", "websocket", "winim", "with", "ws", "yaml", "zero_functional", "zippy" + ] + let ignorePackages = ["rpgsheet", + "arturo", "argument_parser", "murmur", "nimgame", "locale", "nim-locale", + "nim-ao", "ao", "termbox", "linagl", "kwin", "yahooweather", "noaa", + "nimwc", + "artemis"] + let toDownload = importantPackages.filterIt(it notin ignorePackages) + for pkg in toDownload: + echo "Downloading ", pkg + downloadAndStorePackageVersionTableFor(pkg, options) + echo "Done with ", pkg + +suite "SAT solver": + test "can solve simple SAT": + let pkgVersionTable = { + "a": PackageVersions(pkgName: "a", versions: @[ + PackageMinimalInfo(name: "a", version: newVersion "3.0", requires: @[ + (name:"b", ver: parseVersionRange ">= 0.1.0") + ], isRoot:true), + ]), + "b": PackageVersions(pkgName: "b", versions: @[ + PackageMinimalInfo(name: "b", version: newVersion "0.1.0") + ]) + }.toTable() + var graph = pkgVersionTable.toDepGraph() + let form = toFormular(graph) + var packages = initTable[string, Version]() + var output = "" + check solve(graph, form, packages, output) + check packages.len == 2 + check packages["a"] == newVersion "3.0" + check packages["b"] == newVersion "0.1.0" + + + test "solves 'Conflicting dependency resolution' #1162": + let pkgVersionTable = { + "a": PackageVersions(pkgName: "a", versions: @[ + PackageMinimalInfo(name: "a", version: newVersion "3.0", requires: @[ + (name:"b", ver: parseVersionRange ">= 0.1.4"), + (name:"c", ver: parseVersionRange ">= 0.0.5 & <= 0.1.0") + ], isRoot:true), + ]), + "b": PackageVersions(pkgName: "b", versions: @[ + PackageMinimalInfo(name: "b", version: newVersion "0.1.4", requires: @[ + (name:"c", ver: VersionRange(kind: verAny)) + ]), + ]), + "c": PackageVersions(pkgName: "c", versions: @[ + PackageMinimalInfo(name: "c", version: newVersion "0.1.0"), + PackageMinimalInfo(name: "c", version: newVersion "0.2.1") + ]) + }.toTable() + var graph = pkgVersionTable.toDepGraph() + let form = toFormular(graph) + var packages = initTable[string, Version]() + var output = "" + check solve(graph, form, packages, output) + check packages.len == 3 + check packages["a"] == newVersion "3.0" + check packages["b"] == newVersion "0.1.4" + check packages["c"] == newVersion "0.1.0" + + + test "dont solve unsatisfable": + let pkgVersionTable = { + "a": PackageVersions(pkgName: "a", versions: @[ + PackageMinimalInfo(name: "a", version: newVersion "3.0", requires: @[ + (name:"b", ver: parseVersionRange ">= 0.5.0") + ], isRoot:true), + ]), + "b": PackageVersions(pkgName: "b", versions: @[ + PackageMinimalInfo(name: "b", version: newVersion "0.1.0") + ]) + }.toTable() + var graph = pkgVersionTable.toDepGraph() + let form = toFormular(graph) + var packages = initTable[string, Version]() + var output = "" + check not solve(graph, form, packages, output) + echo output + check packages.len == 0 + + #Needs the file structure. Comming in another PR + # test "issue #1162": + # cd "conflictingdepres": + # #integration version of the test above + # #[ + # The folder structure of the test is key for the setup: + # Notice how inside the pkgs2 folder (convention when using local packages) there are 3 folders + # where c has two versions of the same package. The version is retrieved counterintuitively from + # the nimblemeta.json special version field. + # ]# + # let (_, exitCode) = execNimble("install", "-l", "--sat") + # check exitCode == QuitSuccess + + test "should be able to download a package and select its deps": + + let pkgName: string = "nimlangserver" + let pv: PkgTuple = (pkgName, VersionRange(kind: verAny)) + var options = initOptions() + options.nimBin = "nim" + options.config.packageLists["official"] = PackageList(name: "Official", urls: @[ + "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json", + "https://nim-lang.org/nimble/packages.json" + ]) + + var pkgInfo = downloadPkInfoForPv(pv, options) + var root = pkgInfo.getMinimalInfo() + root.isRoot = true + var pkgVersionTable = initTable[string, PackageVersions]() + collectAllVersions(pkgVersionTable, root, options, downloadMinimalPackage) + pkgVersionTable[pkgName] = PackageVersions(pkgName: pkgName, versions: @[root]) + + var graph = pkgVersionTable.toDepGraph() + let form = graph.toFormular() + var packages = initTable[string, Version]() + var output = "" + check solve(graph, form, packages, output) + check packages.len > 0 + + #Needs the file structure. Comming in another PR + # test "should be able to solve all nimble packages": + # # downloadAllPackages() #uncomment this to download all packages. It's better to just keep them cached as it takes a while. + # let now = now() + # var pks = 0 + # for jsonFile in walkPattern("packageMinimal/*.json"): + # inc pks + # var pkgVersionTable = parseJson(readFile(jsonFile)).to(Table[string, PackageVersions]) + # var graph = pkgVersionTable.toDepGraph() + # let form = toFormular(graph) + # var packages = initTable[string, Version]() + # var output = "" + # check solve(graph, form, packages, output) + # check packages.len > 0 + + # let ends = now() + # echo "Solved ", pks, " packages in ", ends - now, " seconds" + + # test "should be able to retrieve the package minimal info from the nimble directory": + # var options = initOptions() + # options.nimbleDir = getCurrentDir() / "conflictingdepres" / "nimbledeps" + # let pkgs = getInstalledMinimalPackages(options) + # var pkgVersionTable = initTable[string, PackageVersions]() + # fillPackageTableFromPreferred(pkgVersionTable, pkgs) + # check pkgVersionTable.hasVersion("b", newVersion "0.1.4") + # check pkgVersionTable.hasVersion("c", newVersion "0.1.0") + # check pkgVersionTable.hasVersion("c", newVersion "0.2.1") + + # test "should fallback to the download if the package is not found in the list of packages": + # let root = + # PackageMinimalInfo( + # name: "a", version: newVersion "3.0", + # requires: @[ + # (name:"b", ver: parseVersionRange ">= 0.1.4"), + # (name:"c", ver: parseVersionRange ">= 0.0.5 & <= 0.1.0"), + # (name: "random", ver: VersionRange(kind: verAny)), + # ], + # isRoot:true) + + # var options = initOptions() + # options.config.packageLists["official"] = PackageList(name: "Official", urls: @[ + # "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json", + # "https://nim-lang.org/nimble/packages.json" + # ]) + # options.nimbleDir = getCurrentDir() / "conflictingdepres" / "nimbledeps" + # options.nimBin = "nim" + # options.pkgCachePath = getCurrentDir() / "conflictingdepres" / "download" + # let pkgs = getInstalledMinimalPackages(options) + # var pkgVersionTable = initTable[string, PackageVersions]() + # pkgVersionTable["a"] = PackageVersions(pkgName: "a", versions: @[root]) + # fillPackageTableFromPreferred(pkgVersionTable, pkgs) + # collectAllVersions(pkgVersionTable, root, options, downloadMinimalPackage) + # var output = "" + # let solvedPkgs = pkgVersionTable.getSolvedPackages(output) + # check solvedPkgs["b"] == newVersion "0.1.4" + # check solvedPkgs["c"] == newVersion "0.1.0" + # check "random" in pkgVersionTable + + # removeDir(options.pkgCachePath) \ No newline at end of file From 6ef0649ceaac6709874157bf7f2bc66df29ee3d0 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 28 Mar 2024 11:22:04 +0000 Subject: [PATCH 2/4] removes import --- src/nimble.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/nimble.nim b/src/nimble.nim index 44d9db5e..a08fbe78 100644 --- a/src/nimble.nim +++ b/src/nimble.nim @@ -1,8 +1,6 @@ # Copyright (C) Dominik Picheta. All rights reserved. # BSD License. Look at license.txt for more info. -import system except TResult - import os, tables, strtabs, json, algorithm, sets, uri, sugar, sequtils, osproc, strformat From 678f2ce628c927e7265f51595945f6b61c240102 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 28 Mar 2024 11:26:13 +0000 Subject: [PATCH 3/4] adds nimble install to tests --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc3586b5..225ef50e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,6 +32,7 @@ jobs: run: nim r src/nimblepkg/private/clone.nim - name: Run nim c -r tester run: | + nimble install cd tests nim c -r tester # there's no need to add nimblepkg unit tests -- From 0747cf8628ac8cea99389b58031925f318e77604 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 4 Apr 2024 16:48:41 +0200 Subject: [PATCH 4/4] Update .github/workflows/test.yml Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6dfee124..19ba7804 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,6 @@ jobs: run: nimble install -y - name: Run nim c -r tester run: | - nimble install cd tests nim c -r tester # there's no need to add nimblepkg unit tests --