Implement parallel package downloads

Parallel downloads of locked packages are implemented. The default
maximum simultaneous downloads are 20. `--max-parallel-downloads` option
is added for setting a user-specified limit. The value 0 is used for no
limit.

Related to #127

src/nimble.nim

@@ -4,7 +4,7 @@
 import system except TResult
 
 import os, tables, strtabs, json, algorithm, sets, uri, sugar, sequtils, osproc,
-       strformat, sequtils
+       strformat, sequtils, asyncdispatch
 
 import std/options as std_opt
 
@@ -446,49 +446,87 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
   cd pkgInfo.myPath.splitFile.dir:
     discard execHook(options, actionInstall, false)
 
-proc getLockedDep(pkgInfo: PackageInfo, name: string, dep: LockFileDep,
-                  options: Options): PackageInfo =
-  ## Returns the package info for dependency package `dep` with name `name` from
-  ## the lock file of the package `pkgInfo` by searching for it in the local
-  ## Nimble cache and downloading it from its repository if the version with the
-  ## required checksum is not found locally.
+proc getDependencyDir(name: string, dep: LockFileDep, options: Options):
+    string =
+  options.getPkgsDir() /  &"{name}-{dep.version}-{dep.checksums.sha1}"
 
-  let packagesDir = options.getPkgsDir()
-  let depDirName = packagesDir / &"{name}-{dep.version}-{dep.checksums.sha1}"
+proc isInstalled(name: string, dep: LockFileDep, options: Options): bool =
+  fileExists(getDependencyDir(name, dep, options) / packageMetaDataFileName)
 
-  if not fileExists(depDirName / packageMetaDataFileName):
-    if depDirName.dirExists:
+proc getDependency(name: string, dep: LockFileDep, options: Options):
+    PackageInfo =
+  let depDirName = getDependencyDir(name, dep, options)
+  let nimbleFilePath = findNimbleFile(depDirName, false)
+  getInstalledPackageMin(depDirName, nimbleFilePath).toFullInfo(options)
+
+type
+  DownloadInfo = ref object
+    dependency: LockFileDep
+    url: string
+    version: VersionRange
+    downloadDir: string
+    vcsRevision: Sha1HashRef
+
+  DownloadQueue = ref seq[tuple[name: string, dep: LockFileDep]]
+  DownloadResults = ref seq[DownloadInfo]
+
+proc downloadDependency(name: string, dep: LockFileDep, options: Options):
+    Future[DownloadInfo] {.async.} =
+  let depDirName = getDependencyDir(name, dep, options)
+
+  if depDirName.dirExists:
       promptRemoveEntirePackageDir(depDirName, options)
       removeDir(depDirName)
 
-    let (url, metadata) = getUrlData(dep.url)
-    let version =  dep.version.parseVersionRange
-    let subdir = metadata.getOrDefault("subdir")
+  let (url, metadata) = getUrlData(dep.url)
+  let version =  dep.version.parseVersionRange
+  let subdir = metadata.getOrDefault("subdir")
 
-    let (downloadDir, _, vcsRevision) = downloadPkg(
-      url, version, dep.downloadMethod, subdir, options,
-      downloadPath = "", dep.vcsRevision)
+  let (downloadDir, _, vcsRevision) = await downloadPkg(
+    url, version, dep.downloadMethod, subdir, options,
+    downloadPath = "", dep.vcsRevision)
 
-    let downloadedPackageChecksum = calculateDirSha1Checksum(downloadDir)
-    if downloadedPackageChecksum != dep.checksums.sha1:
-      raise checksumError(name, dep.version, dep.vcsRevision,
-                          downloadedPackageChecksum, dep.checksums.sha1)
+  let downloadedPackageChecksum = calculateDirSha1Checksum(downloadDir)
+  if downloadedPackageChecksum != dep.checksums.sha1:
+    raise checksumError(name, dep.version, dep.vcsRevision,
+                        downloadedPackageChecksum, dep.checksums.sha1)
 
-    var (_, newlyInstalledPackageInfo) = installFromDir(downloadDir, version,
-      options, url, first = false, fromLockFile = true, vcsRevision)
+  result = DownloadInfo(
+    dependency: dep,
+    url: url, 
+    version: version,
+    downloadDir: downloadDir,
+    vcsRevision: vcsRevision)
 
-    for depDepName in dep.dependencies:
-      let depDep = pkgInfo.lockedDeps[depDepName]
-      let revDep = (name: depDepName, version: depDep.version,
-                    checksum: depDep.checksums.sha1) 
-      options.nimbleData.addRevDep(revDep, newlyInstalledPackageInfo)
+proc installDependency(pkgInfo: PackageInfo, downloadInfo: DownloadInfo,
+                       options: Options): PackageInfo =
 
-    return newlyInstalledPackageInfo
-  else:
-    let nimbleFilePath = findNimbleFile(depDirName, false)
-    let packageInfo = getInstalledPackageMin(
-      depDirName, nimbleFilePath).toFullInfo(options)
-    return packageInfo
+  let (_, newlyInstalledPkgInfo) = installFromDir(
+    downloadInfo.downloadDir,
+    downloadInfo.version,
+    options,
+    downloadInfo.url,
+    first = false,
+    fromLockFile = true,
+    downloadInfo.vcsRevision[])
+
+  downloadInfo.downloadDir.removeDir
+
+  for depDepName in downloadInfo.dependency.dependencies:
+    let depDep = pkgInfo.lockedDeps[depDepName]
+    let revDep = (name: depDepName, version: depDep.version,
+                  checksum: depDep.checksums.sha1) 
+    options.nimbleData.addRevDep(revDep, newlyInstalledPkgInfo)
+
+  return newlyInstalledPkgInfo
+
+proc startDownloadWorker(queue: DownloadQueue, options: Options,
+                         downloadResults: DownloadResults) {.async.} =
+  while queue[].len > 0:
+    let download = queue[].pop
+    let index = queue[].len
+    downloadResults[index] = await downloadDependency(
+      download.name, download.dep, options)
 
 proc processLockedDependencies(pkgInfo: PackageInfo, options: Options):
     HashSet[PackageInfo] =
@@ -499,14 +537,30 @@ proc processLockedDependencies(pkgInfo: PackageInfo, options: Options):
   # installs it by downloading it from its repository.
 
   let developModeDeps = getDevelopDependencies(pkgInfo, options)
+
+  var dependenciesToDownload: DownloadQueue
+  dependenciesToDownload.new
+
   for name, dep in pkgInfo.lockedDeps:
-    let depPkg =
-      if developModeDeps.hasKey(name):
-        developModeDeps[name][]
-      else:
-        getLockedDep(pkgInfo, name, dep, options)
+    if developModeDeps.hasKey(name):
+      result.incl developModeDeps[name][]
+    elif isInstalled(name, dep, options):
+      result.incl getDependency(name, dep, options)
+    else:
+      dependenciesToDownload[].add (name, dep)
+
+  var downloadResults: DownloadResults
+  downloadResults.new
+  downloadResults[].setLen(dependenciesToDownload[].len)
+
+  var downloadWorkers: seq[Future[void]]
+  for i in 0 ..< options.maxParallelDownloads:
+    downloadWorkers.add startDownloadWorker(
+      dependenciesToDownload, options, downloadResults)
+  waitFor all(downloadWorkers)
 
-    result.incl depPkg
+  for downloadResult in downloadResults[]:
+    result.incl installDependency(pkgInfo, downloadResult, options)
 
 proc getDownloadInfo*(pv: PkgTuple, options: Options,
                       doPrompt: bool): (DownloadMethod, string,
@@ -555,11 +609,11 @@ proc install(packages: seq[PkgTuple], options: Options,
       let (meth, url, metadata) = getDownloadInfo(pv, options, doPrompt)
       let subdir = metadata.getOrDefault("subdir")
       let (downloadDir, downloadVersion, vcsRevision) =
-          downloadPkg(url, pv.ver, meth, subdir, options, downloadPath = "",
-                      vcsRevision = notSetSha1Hash)
+          waitFor downloadPkg(url, pv.ver, meth, subdir, options,
+                              downloadPath = "", vcsRevision = notSetSha1Hash)
       try:
         result = installFromDir(downloadDir, pv.ver, options, url,
-                                first, fromLockFile, vcsRevision)
+                                first, fromLockFile, vcsRevision[])
       except BuildFailed as error:
         # The package failed to build.
         # Check if we tried building a tagged version of the package.
@@ -1107,8 +1161,8 @@ proc installDevelopPackage(pkgTup: PkgTuple, options: Options): string =
 
   var options = options
   options.forceFullClone = true
-  discard downloadPkg(url, ver, meth, subdir, options, downloadDir,
-                      vcsRevision = notSetSha1Hash)
+  discard waitFor downloadPkg(url, ver, meth, subdir, options, downloadDir,
+                              vcsRevision = notSetSha1Hash)
 
   let pkgDir = downloadDir / subdir
   var pkgInfo = getPkgInfo(pkgDir, options)