From f38feb15935038f1e28b5882e0f9bd4adf1f56b3 Mon Sep 17 00:00:00 2001 From: Ivan Bobev Date: Tue, 1 Jun 2021 21:30:49 +0300 Subject: [PATCH] Implement parallel package downloads Parallel downloads of locked packages are implemented. The default maximum simultaneous downloads are 20. `--max-parallel-downloads` option is added for setting a user-specified limit. The value 0 is used for no limit. Related to nim-lang/nimble#127 --- src/nimble.nim | 144 +++++++++++++++++++--------- src/nimblepkg/download.nim | 179 ++++++++++++++++++----------------- src/nimblepkg/options.nim | 6 ++ src/nimblepkg/sha1hashes.nim | 2 + src/nimblepkg/tools.nim | 17 +++- 5 files changed, 217 insertions(+), 131 deletions(-) diff --git a/src/nimble.nim b/src/nimble.nim index 06a85f1c1..4f10d33bb 100644 --- a/src/nimble.nim +++ b/src/nimble.nim @@ -4,7 +4,7 @@ import system except TResult import os, tables, strtabs, json, algorithm, sets, uri, sugar, sequtils, osproc, - strformat, sequtils + strformat, sequtils, asyncdispatch import std/options as std_opt @@ -446,49 +446,87 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options, cd pkgInfo.myPath.splitFile.dir: discard execHook(options, actionInstall, false) -proc getLockedDep(pkgInfo: PackageInfo, name: string, dep: LockFileDep, - options: Options): PackageInfo = - ## Returns the package info for dependency package `dep` with name `name` from - ## the lock file of the package `pkgInfo` by searching for it in the local - ## Nimble cache and downloading it from its repository if the version with the - ## required checksum is not found locally. +proc getDependencyDir(name: string, dep: LockFileDep, options: Options): + string = + options.getPkgsDir() / &"{name}-{dep.version}-{dep.checksums.sha1}" - let packagesDir = options.getPkgsDir() - let depDirName = packagesDir / &"{name}-{dep.version}-{dep.checksums.sha1}" +proc isInstalled(name: string, dep: LockFileDep, options: Options): bool = + fileExists(getDependencyDir(name, dep, options) / packageMetaDataFileName) - if not fileExists(depDirName / packageMetaDataFileName): - if depDirName.dirExists: +proc getDependency(name: string, dep: LockFileDep, options: Options): + PackageInfo = + let depDirName = getDependencyDir(name, dep, options) + let nimbleFilePath = findNimbleFile(depDirName, false) + getInstalledPackageMin(depDirName, nimbleFilePath).toFullInfo(options) + +type + DownloadInfo = ref object + dependency: LockFileDep + url: string + version: VersionRange + downloadDir: string + vcsRevision: Sha1HashRef + + DownloadQueue = ref seq[tuple[name: string, dep: LockFileDep]] + DownloadResults = ref seq[DownloadInfo] + +proc downloadDependency(name: string, dep: LockFileDep, options: Options): + Future[DownloadInfo] {.async.} = + let depDirName = getDependencyDir(name, dep, options) + + if depDirName.dirExists: promptRemoveEntirePackageDir(depDirName, options) removeDir(depDirName) - let (url, metadata) = getUrlData(dep.url) - let version = dep.version.parseVersionRange - let subdir = metadata.getOrDefault("subdir") + let (url, metadata) = getUrlData(dep.url) + let version = dep.version.parseVersionRange + let subdir = metadata.getOrDefault("subdir") - let (downloadDir, _, vcsRevision) = downloadPkg( - url, version, dep.downloadMethod, subdir, options, - downloadPath = "", dep.vcsRevision) + let (downloadDir, _, vcsRevision) = await downloadPkg( + url, version, dep.downloadMethod, subdir, options, + downloadPath = "", dep.vcsRevision) - let downloadedPackageChecksum = calculateDirSha1Checksum(downloadDir) - if downloadedPackageChecksum != dep.checksums.sha1: - raise checksumError(name, dep.version, dep.vcsRevision, - downloadedPackageChecksum, dep.checksums.sha1) + let downloadedPackageChecksum = calculateDirSha1Checksum(downloadDir) + if downloadedPackageChecksum != dep.checksums.sha1: + raise checksumError(name, dep.version, dep.vcsRevision, + downloadedPackageChecksum, dep.checksums.sha1) - var (_, newlyInstalledPackageInfo) = installFromDir(downloadDir, version, - options, url, first = false, fromLockFile = true, vcsRevision) + result = DownloadInfo( + dependency: dep, + url: url, + version: version, + downloadDir: downloadDir, + vcsRevision: vcsRevision) - for depDepName in dep.dependencies: - let depDep = pkgInfo.lockedDeps[depDepName] - let revDep = (name: depDepName, version: depDep.version, - checksum: depDep.checksums.sha1) - options.nimbleData.addRevDep(revDep, newlyInstalledPackageInfo) +proc installDependency(pkgInfo: PackageInfo, downloadInfo: DownloadInfo, + options: Options): PackageInfo = - return newlyInstalledPackageInfo - else: - let nimbleFilePath = findNimbleFile(depDirName, false) - let packageInfo = getInstalledPackageMin( - depDirName, nimbleFilePath).toFullInfo(options) - return packageInfo + let (_, newlyInstalledPkgInfo) = installFromDir( + downloadInfo.downloadDir, + downloadInfo.version, + options, + downloadInfo.url, + first = false, + fromLockFile = true, + downloadInfo.vcsRevision[]) + + downloadInfo.downloadDir.removeDir + + for depDepName in downloadInfo.dependency.dependencies: + let depDep = pkgInfo.lockedDeps[depDepName] + let revDep = (name: depDepName, version: depDep.version, + checksum: depDep.checksums.sha1) + options.nimbleData.addRevDep(revDep, newlyInstalledPkgInfo) + + return newlyInstalledPkgInfo + +proc startDownloadWorker(queue: DownloadQueue, options: Options, + downloadResults: DownloadResults) {.async.} = + while queue[].len > 0: + let download = queue[].pop + let index = queue[].len + downloadResults[index] = await downloadDependency( + download.name, download.dep, options) proc processLockedDependencies(pkgInfo: PackageInfo, options: Options): HashSet[PackageInfo] = @@ -499,14 +537,30 @@ proc processLockedDependencies(pkgInfo: PackageInfo, options: Options): # installs it by downloading it from its repository. let developModeDeps = getDevelopDependencies(pkgInfo, options) + + var dependenciesToDownload: DownloadQueue + dependenciesToDownload.new + for name, dep in pkgInfo.lockedDeps: - let depPkg = - if developModeDeps.hasKey(name): - developModeDeps[name][] - else: - getLockedDep(pkgInfo, name, dep, options) + if developModeDeps.hasKey(name): + result.incl developModeDeps[name][] + elif isInstalled(name, dep, options): + result.incl getDependency(name, dep, options) + else: + dependenciesToDownload[].add (name, dep) + + var downloadResults: DownloadResults + downloadResults.new + downloadResults[].setLen(dependenciesToDownload[].len) + + var downloadWorkers: seq[Future[void]] + for i in 0 ..< options.maxParallelDownloads: + downloadWorkers.add startDownloadWorker( + dependenciesToDownload, options, downloadResults) + waitFor all(downloadWorkers) - result.incl depPkg + for downloadResult in downloadResults[]: + result.incl installDependency(pkgInfo, downloadResult, options) proc getDownloadInfo*(pv: PkgTuple, options: Options, doPrompt: bool): (DownloadMethod, string, @@ -555,11 +609,11 @@ proc install(packages: seq[PkgTuple], options: Options, let (meth, url, metadata) = getDownloadInfo(pv, options, doPrompt) let subdir = metadata.getOrDefault("subdir") let (downloadDir, downloadVersion, vcsRevision) = - downloadPkg(url, pv.ver, meth, subdir, options, downloadPath = "", - vcsRevision = notSetSha1Hash) + waitFor downloadPkg(url, pv.ver, meth, subdir, options, + downloadPath = "", vcsRevision = notSetSha1Hash) try: result = installFromDir(downloadDir, pv.ver, options, url, - first, fromLockFile, vcsRevision) + first, fromLockFile, vcsRevision[]) except BuildFailed as error: # The package failed to build. # Check if we tried building a tagged version of the package. @@ -1107,8 +1161,8 @@ proc installDevelopPackage(pkgTup: PkgTuple, options: Options): string = var options = options options.forceFullClone = true - discard downloadPkg(url, ver, meth, subdir, options, downloadDir, - vcsRevision = notSetSha1Hash) + discard waitFor downloadPkg(url, ver, meth, subdir, options, downloadDir, + vcsRevision = notSetSha1Hash) let pkgDir = downloadDir / subdir var pkgInfo = getPkgInfo(pkgDir, options) diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index 0b4766512..4a3cad79e 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -2,7 +2,7 @@ # BSD License. Look at license.txt for more info. import parseutils, os, osproc, strutils, tables, pegs, uri, strformat, - httpclient, json + httpclient, json, asyncdispatch from algorithm import SortOrder, sorted from sequtils import toSeq, filterIt, map @@ -10,55 +10,51 @@ from sequtils import toSeq, filterIt, map import packageinfotypes, packageparser, version, tools, common, options, cli, sha1hashes, vcstools -proc doCheckout(meth: DownloadMethod, downloadDir, branch: string) = - case meth - of DownloadMethod.git: - cd downloadDir: - # Force is used here because local changes may appear straight after a - # clone has happened. Like in the case of git on Windows where it - # messes up the damn line endings. - doCmd("git checkout --force " & branch) - doCmd("git submodule update --recursive --depth 1") - of DownloadMethod.hg: - cd downloadDir: - doCmd("hg checkout " & branch) +type + DownloadPkgResult = tuple + dir: string + version: Version + vcsRevision: Sha1HashRef -proc doPull(meth: DownloadMethod, downloadDir: string) {.used.} = +proc doCheckout(meth: DownloadMethod, downloadDir, branch: string): + Future[void] {.async.} = case meth of DownloadMethod.git: - doCheckout(meth, downloadDir, "") - cd downloadDir: - doCmd("git pull") - if fileExists(".gitmodules"): - doCmd("git submodule update --recursive --depth 1") + # Force is used here because local changes may appear straight after a clone + # has happened. Like in the case of git on Windows where it messes up the + # damn line endings. + discard await tryDoCmdExAsync( + &"git -C {downloadDir} checkout --force {branch}") + discard await tryDoCmdExAsync( + &"git -C {downloadDir} submodule update --recursive --depth 1") of DownloadMethod.hg: - doCheckout(meth, downloadDir, "default") - cd downloadDir: - doCmd("hg pull") + discard await tryDoCmdExAsync(&"hg --cwd {downloadDir} checkout {branch}") proc doClone(meth: DownloadMethod, url, downloadDir: string, branch = "", - onlyTip = true) = + onlyTip = true) {.async.} = case meth of DownloadMethod.git: let - depthArg = if onlyTip: "--depth 1 " else: "" - branchArg = if branch == "": "" else: "-b " & branch & " " - doCmd("git clone --recursive " & depthArg & branchArg & - url & " " & downloadDir) + depthArg = if onlyTip: "--depth 1" else: "" + branchArg = if branch == "": "" else: "-b " & branch + discard await tryDoCmdExAsync( + &"git clone --recursive {depthArg} {branchArg} {url} {downloadDir}") of DownloadMethod.hg: let - tipArg = if onlyTip: "-r tip " else: "" - branchArg = if branch == "": "" else: "-b " & branch & " " - doCmd("hg clone " & tipArg & branchArg & url & " " & downloadDir) + tipArg = if onlyTip: "-r tip" else: "" + branchArg = if branch == "": "" else: "-b " & branch + discard await tryDoCmdExAsync( + &"hg clone {tipArg} {branchArg} {url} {downloadDir}") -proc getTagsList(dir: string, meth: DownloadMethod): seq[string] = +proc getTagsList(dir: string, meth: DownloadMethod): + Future[seq[string]] {.async.} = var output: string cd dir: case meth of DownloadMethod.git: - output = execProcess("git tag") + output = await tryDoCmdExAsync("git tag") of DownloadMethod.hg: - output = execProcess("hg tags") + output = await tryDoCmdExAsync("hg tags") if output.len > 0: case meth of DownloadMethod.git: @@ -77,11 +73,13 @@ proc getTagsList(dir: string, meth: DownloadMethod): seq[string] = else: result = @[] -proc getTagsListRemote*(url: string, meth: DownloadMethod): seq[string] = +proc getTagsListRemote*(url: string, meth: DownloadMethod): + Future[seq[string]] {.async.} = result = @[] case meth of DownloadMethod.git: - var (output, exitCode) = doCmdEx("git ls-remote --tags " & url.quoteShell()) + var (output, exitCode) = await doCmdExAsync( + &"git ls-remote --tags {url.quoteShell}") if exitCode != QuitSuccess: raise nimbleError("Unable to query remote tags for " & url & ". Git returned: " & output) @@ -143,19 +141,24 @@ proc isURL*(name: string): bool = name.startsWith(peg" @'://' ") proc cloneSpecificRevision(downloadMethod: DownloadMethod, - url, downloadDir: string, vcsRevision: Sha1Hash) = + url, downloadDir: string, + vcsRevision: Sha1Hash) {.async.} = assert vcsRevision != notSetSha1Hash display("Cloning", "revision: " & $vcsRevision, priority = MediumPriority) case downloadMethod of DownloadMethod.git: + let downloadDir = downloadDir.quoteShell createDir(downloadDir) - cd downloadDir: - doCmd("git init") - doCmd(fmt"git remote add origin {url}") - doCmd(fmt"git fetch --depth 1 origin {vcsRevision}") - doCmd("git reset --hard FETCH_HEAD") + discard await tryDoCmdExAsync( + &"git -C {downloadDir} init") + discard await tryDoCmdExAsync( + &"git -C {downloadDir} remote add origin {url}") + discard await tryDoCmdExAsync( + &"git -C {downloadDir} fetch --depth 1 origin {vcsRevision}") + discard await tryDoCmdExAsync( + &"git -C {downloadDir} reset --hard FETCH_HEAD") of DownloadMethod.hg: - doCmd(fmt"hg clone {url} -r {vcsRevision}") + discard await tryDoCmdExAsync(&"hg clone {url} -r {vcsRevision}") proc getTarExePath: string = ## Returns path to `tar` executable. @@ -226,23 +229,23 @@ proc getGitHubApiUrl(url, commit: string): string = ## an URL for the GitHub REST API query for the full commit hash. &"https://api.github.com/repos/{extractOwnerAndRepo(url)}/commits/{commit}" -proc getUrlContent(url: string): string = +proc getUrlContent(url: string): Future[string] {.async.} = ## Makes a GET request to `url`. - var client {.global.}: HttpClient - once: client = newHttpClient() - return client.getContent(url) + let client = newAsyncHttpClient() + return await client.getContent(url) {.warning[ProveInit]: off.} -proc getFullRevisionFromGitHubApi(url, version: string): Sha1Hash = +proc getFullRevisionFromGitHubApi(url, version: string): + Future[Sha1HashRef] {.async.} = ## By given a commit short hash and an URL to a GitHub repository retrieves ## the full hash of the commit by using GitHub REST API. try: let gitHubApiUrl = getGitHubApiUrl(url, version) display("Get", gitHubApiUrl); - let content = getUrlContent(gitHubApiUrl) + let content = await getUrlContent(gitHubApiUrl) let json = parseJson(content) if json.hasKey("sha"): - return json["sha"].str.initSha1Hash + return json["sha"].str.initSha1Hash.newClone else: raise nimbleError(json["message"].str) except CatchableError as error: @@ -250,7 +253,7 @@ proc getFullRevisionFromGitHubApi(url, version: string): Sha1Hash = &"of package at \"{url}\".", details = error) {.warning[ProveInit]: on.} -proc parseRevision(lsRemoteOutput: string): Sha1Hash = +proc parseRevision(lsRemoteOutput: string): Sha1HashRef = ## Parses the output from `git ls-remote` call to extract the returned sha1 ## hash value. Even when successful the first line of the command's output ## can be a redirection warning. @@ -258,19 +261,19 @@ proc parseRevision(lsRemoteOutput: string): Sha1Hash = for line in lines: if line.len >= 40: try: - return initSha1Hash(line[0..39]) + return line[0..39].initSha1Hash.newClone except InvalidSha1HashError: discard - return notSetSha1Hash + return notSetSha1Hash.newClone -proc getRevision(url, version: string): Sha1Hash = +proc getRevision(url, version: string): Future[Sha1HashRef] {.async.} = ## Returns the commit hash corresponding to the given `version` of the package ## in repository at `url`. - let output = tryDoCmdEx(&"git ls-remote {url} {version}") + let output = await tryDoCmdExAsync(&"git ls-remote {url} {version}") result = parseRevision(output) - if result == notSetSha1Hash: + if result[] == notSetSha1Hash: if version.seemsLikeRevision: - result = getFullRevisionFromGitHubApi(url, version) + result = await getFullRevisionFromGitHubApi(url, version) else: raise nimbleError(&"Cannot get revision for version \"{version}\" " & &"of package at \"{url}\".") @@ -286,13 +289,13 @@ proc getTarCmdLine(downloadDir, filePath: string): string = &"tar -C {downloadDir} -xf {filePath} --strip-components 1" proc doDownloadTarball(url, downloadDir, version: string, queryRevision: bool): - Sha1Hash = + Future[Sha1HashRef] {.async.} = ## Downloads package tarball from GitHub. Returns the commit hash of the ## downloaded package in the case `queryRevision` is `true`. let downloadLink = getTarballDownloadLink(url, version) display("Downloading", downloadLink) - let data = getUrlContent(downloadLink) + let data = await getUrlContent(downloadLink) display("Completed", "downloading " & downloadLink) let filePath = downloadDir / "tarball.tar.gz" @@ -303,7 +306,7 @@ proc doDownloadTarball(url, downloadDir, version: string, queryRevision: bool): display("Unpacking", filePath) let cmd = getTarCmdLine(downloadDir, filePath) - let (output, exitCode) = doCmdEx(cmd) + let (output, exitCode) = await doCmdExAsync(cmd) if exitCode != QuitSuccess and not output.contains("Cannot create symlink to"): # If the command fails for reason different then unable establishing a # sym-link raise an exception. This reason for failure is common on Windows @@ -314,13 +317,14 @@ proc doDownloadTarball(url, downloadDir, version: string, queryRevision: bool): display("Completed", "unpacking " & filePath) filePath.removeFile - return if queryRevision: getRevision(url, version) else: notSetSha1Hash + return if queryRevision: await getRevision(url, version) + else: notSetSha1Hash.newClone {.warning[ProveInit]: off.} proc doDownload(url: string, downloadDir: string, verRange: VersionRange, downMethod: DownloadMethod, options: Options, vcsRevision: Sha1Hash): - tuple[version: Version, vcsRevision: Sha1Hash] = + Future[tuple[version: Version, vcsRevision: Sha1HashRef]] {.async.} = ## Downloads the repository specified by ``url`` using the specified download ## method. ## @@ -338,36 +342,38 @@ proc doDownload(url: string, downloadDir: string, verRange: VersionRange, if $latest.ver != "": result.version = latest.ver - result.vcsRevision = notSetSha1Hash + result.vcsRevision = notSetSha1Hash.newClone removeDir(downloadDir) if vcsRevision != notSetSha1Hash: if downloadTarball(url, options): - discard doDownloadTarball(url, downloadDir, $vcsRevision, false) + discard await doDownloadTarball(url, downloadDir, $vcsRevision, false) else: - cloneSpecificRevision(downMethod, url, downloadDir, vcsRevision) - result.vcsRevision = vcsRevision + await cloneSpecificRevision(downMethod, url, downloadDir, vcsRevision) + result.vcsRevision = vcsRevision.newClone elif verRange.kind == verSpecial: # We want a specific commit/branch/tag here. if verRange.spe == getHeadName(downMethod): # Grab HEAD. if downloadTarball(url, options): - result.vcsRevision = doDownloadTarball(url, downloadDir, "HEAD", true) + result.vcsRevision = await doDownloadTarball( + url, downloadDir, "HEAD", true) else: - doClone(downMethod, url, downloadDir, onlyTip = not options.forceFullClone) + await doClone(downMethod, url, downloadDir, + onlyTip = not options.forceFullClone) else: assert ($verRange.spe)[0] == '#', "The special version must start with '#'." let specialVersion = substr($verRange.spe, 1) if downloadTarball(url, options): - result.vcsRevision = doDownloadTarball( + result.vcsRevision = await doDownloadTarball( url, downloadDir, specialVersion, true) else: # Grab the full repo. - doClone(downMethod, url, downloadDir, onlyTip = false) + await doClone(downMethod, url, downloadDir, onlyTip = false) # Then perform a checkout operation to get the specified branch/commit. # `spe` starts with '#', trim it. - doCheckout(downMethod, downloadDir, specialVersion) + await doCheckout(downMethod, downloadDir, specialVersion) result.version = verRange.spe else: case downMethod @@ -375,40 +381,43 @@ proc doDownload(url: string, downloadDir: string, verRange: VersionRange, # For Git we have to query the repo remotely for its tags. This is # necessary as cloning with a --depth of 1 removes all tag info. result.version = getHeadName(downMethod) - let versions = getTagsListRemote(url, downMethod).getVersionList() + let versions = (await getTagsListRemote(url, downMethod)).getVersionList() if versions.len > 0: getLatestByTag: if downloadTarball(url, options): let versionToDownload = if latest.tag.len > 0: latest.tag else: "HEAD" - result.vcsRevision = doDownloadTarball( + result.vcsRevision = await doDownloadTarball( url, downloadDir, versionToDownload, true) else: display("Cloning", "latest tagged version: " & latest.tag, priority = MediumPriority) - doClone(downMethod, url, downloadDir, latest.tag, - onlyTip = not options.forceFullClone) + await doClone(downMethod, url, downloadDir, latest.tag, + onlyTip = not options.forceFullClone) else: if downloadTarball(url, options): - result.vcsRevision = doDownloadTarball(url, downloadDir, "HEAD", true) + result.vcsRevision = await doDownloadTarball( + url, downloadDir, "HEAD", true) else: # If no commits have been tagged on the repo we just clone HEAD. - doClone(downMethod, url, downloadDir) # Grab HEAD. + await doClone(downMethod, url, downloadDir) # Grab HEAD. of DownloadMethod.hg: - doClone(downMethod, url, downloadDir, onlyTip = not options.forceFullClone) + await doClone(downMethod, url, downloadDir, + onlyTip = not options.forceFullClone) result.version = getHeadName(downMethod) - let versions = getTagsList(downloadDir, downMethod).getVersionList() + let versions = + (await getTagsList(downloadDir, downMethod)).getVersionList() if versions.len > 0: getLatestByTag: display("Switching", "to latest tagged version: " & latest.tag, priority = MediumPriority) - doCheckout(downMethod, downloadDir, latest.tag) + await doCheckout(downMethod, downloadDir, latest.tag) - if result.vcsRevision == notSetSha1Hash: + if result.vcsRevision[] == notSetSha1Hash: # In the case the package in not downloaded as tarball we must query its # VCS revision from its download directory. - result.vcsRevision = getVcsRevision(downloadDir) + result.vcsRevision = downloadDir.getVcsRevision.newClone {.warning[ProveInit]: on.} proc downloadPkg*(url: string, verRange: VersionRange, @@ -416,8 +425,7 @@ proc downloadPkg*(url: string, verRange: VersionRange, subdir: string, options: Options, downloadPath: string, - vcsRevision: Sha1Hash): - tuple[dir: string, version: Version, vcsRevision: Sha1Hash] = + vcsRevision: Sha1Hash): Future[DownloadPkgResult] {.async.} = ## Downloads the repository as specified by ``url`` and ``verRange`` using ## the download method specified. ## @@ -460,7 +468,7 @@ proc downloadPkg*(url: string, verRange: VersionRange, priority = HighPriority) result.dir = downloadDir / subdir - (result.version, result.vcsRevision) = doDownload( + (result.version, result.vcsRevision) = await doDownload( modUrl, downloadDir, verRange, downMethod, options, vcsRevision) if verRange.kind != verSpecial: @@ -478,7 +486,8 @@ proc echoPackageVersions*(pkg: Package) = case downMethod of DownloadMethod.git: try: - let versions = getTagsListRemote(pkg.url, downMethod).getVersionList() + let versions = + (waitFor getTagsListRemote(pkg.url, downMethod)).getVersionList() if versions.len > 0: let sortedVersions = toSeq(values(versions)) echo(" versions: " & join(sortedVersions, ", ")) diff --git a/src/nimblepkg/options.nim b/src/nimblepkg/options.nim index 983f773ee..94a963f2b 100644 --- a/src/nimblepkg/options.nim +++ b/src/nimblepkg/options.nim @@ -42,6 +42,8 @@ type developLocaldeps*: bool # True if local deps + nimble develop pkg1 ... disableSslCertCheck*: bool noTarballs*: bool # Disable downloading of packages as tarballs from GitHub. + maxParallelDownloads*: int # This is the maximum number of parallel + # downloads. 0 means no limit. ActionType* = enum actionNil, actionRefresh, actionInit, actionDump, actionPublish, @@ -175,6 +177,8 @@ Nimble Options: -l, --localdeps Run in project local dependency mode -t, --no-tarballs Disable downloading of packages as tarballs when working with GitHub repositories. + -m, --max-parallel-downloads The maximum number of parallel downloads. + The default value is 20. Use 0 for no limit. --ver Query remote server for package version information when searching or listing packages. --nimbleDir:dirname Set the Nimble directory. @@ -461,6 +465,7 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) = of "localdeps", "l": result.localdeps = true of "nosslcheck": result.disableSslCertCheck = true of "no-tarballs", "t": result.noTarballs = true + of "max-parallel-downloads", "m": result.maxParallelDownloads = parseInt(val) else: isGlobalFlag = false var wasFlagHandled = true @@ -558,6 +563,7 @@ proc initOptions*(): Options = verbosity: HighPriority, noColor: not isatty(stdout), startDir: getCurrentDir(), + maxParallelDownloads: 20, ) proc handleUnknownFlags(options: var Options) = diff --git a/src/nimblepkg/sha1hashes.nim b/src/nimblepkg/sha1hashes.nim index 5c12cdb6f..497052fbe 100644 --- a/src/nimblepkg/sha1hashes.nim +++ b/src/nimblepkg/sha1hashes.nim @@ -13,6 +13,8 @@ type ## procedure which validates the input. hashValue: string + Sha1HashRef* = ref Sha1Hash + const notSetSha1Hash* = Sha1Hash(hashValue: "") diff --git a/src/nimblepkg/tools.nim b/src/nimblepkg/tools.nim index 7e65175b2..a1286f6ee 100644 --- a/src/nimblepkg/tools.nim +++ b/src/nimblepkg/tools.nim @@ -3,10 +3,12 @@ # # Various miscellaneous utility functions reside here. import osproc, pegs, strutils, os, uri, sets, json, parseutils, strformat, - sequtils + sequtils, asyncdispatch + from net import SslCVerifyMode, newContext, SslContext import version, cli, common, packageinfotypes, options, sha1hashes +import asynctools/asyncproc except quoteShell from compiler/nimblecmd import getPathVersionChecksum proc extractBin(cmd: string): string = @@ -50,6 +52,13 @@ proc doCmdEx*(cmd: string): ProcessOutput = raise nimbleError("'" & bin & "' not in PATH.") return execCmdEx(cmd) +proc doCmdExAsync*(cmd: string): Future[ProcessOutput] {.async.} = + let bin = extractBin(cmd) + if findExe(bin) == "": + raise nimbleError("'" & bin & "' not in PATH.") + let res = await asyncproc.execProcess(cmd) + return (res.output, res.exitCode) + proc tryDoCmdExErrorMessage*(cmd, output: string, exitCode: int): string = &"Execution of '{cmd}' failed with an exit code {exitCode}.\n" & &"Details: {output}" @@ -60,6 +69,12 @@ proc tryDoCmdEx*(cmd: string): string {.discardable.} = raise nimbleError(tryDoCmdExErrorMessage(cmd, output, exitCode)) return output +proc tryDoCmdExAsync*(cmd: string): Future[string] {.async.} = + let (output, exitCode) = await doCmdExAsync(cmd) + if exitCode != QuitSuccess: + raise nimbleError(tryDoCmdExErrorMessage(cmd, output, exitCode)) + return output + proc getNimBin*: string = result = "nim" if findExe("nim") != "": result = findExe("nim")