Merge pull request #60 from bestander/npm-offline-mirror-support

Npm offline mirror support

Author: bestander

package.json

@@ -11,6 +11,7 @@
     "cmd-shim": "^2.0.1",
     "commander": "^2.9.0",
     "diff": "^2.2.1",
+    "ini": "^1.3.4",
     "invariant": "^2.2.0",
     "is-builtin-module": "^1.0.0",
     "kreporters": "^1.0.2",

src/cli/commands/install.js

@@ -93,8 +93,6 @@ export class Install {
     let patterns = [];
     let deps = [];
 
-    let foundConfig = false;
-
     for (let registry of Object.keys(registries)) {
       let filenames = registries[registry].filenames;
 
@@ -103,7 +101,6 @@ export class Install {
         if (!(await fs.exists(loc))) continue;
 
         this.registries.push(registry);
-        foundConfig = true;
 
         let json = await fs.readJson(loc);
         Object.assign(this.resolutions, json.resolutions);
@@ -159,15 +156,13 @@ export class Install {
     let totalStepsThis = totalSteps + 2;
 
     this.reporter.step(1, totalStepsThis, "Rehydrating dependency graph", emoji.get("fast_forward"));
-    await this.resolver.init(requests, true);
-
-    patterns = await this.flatten(patterns);
+    await this.resolver.init(requests);
 
     this.reporter.step(2, totalStepsThis, "Fetching packages", emoji.get("package"));
     await this.resolver.fetcher.init();
 
     return {
-      patterns,
+      patterns: await this.flatten(patterns),
       total: totalStepsThis,
       step: 2
     };
@@ -231,7 +226,7 @@ export class Install {
    * Save added packages to `package.json` if any of the --save flags were used
    */
 
-  async savePackages(patterns: Array<string>) {
+  async savePackages(patterns: Array<string>): Promise<void> {
     if (!this.args.length) return;
 
     let { save, saveDev, saveExact, saveOptional } = this.flags;
@@ -417,7 +412,8 @@ export async function run(
     throw new MessageError("Missing package names for --save flags");
   }
 
-  let lockfile = await Lockfile.fromDirectory(config.cwd, reporter, isStrictLockfile(flags, args));
+  let lockfile = await Lockfile.fromDirectory(config.cwd, reporter, isStrictLockfile(flags, args),
+    hasSaveFlags(flags));
   let install = new Install("install", flags, args, config, reporter, lockfile);
   return install.init();
 }

src/config.js

@@ -24,6 +24,7 @@ import map from "./util/map.js";
 let invariant = require("invariant");
 let userHome  = require("user-home");
 let path      = require("path");
+let url       = require("url");
 
 type ConfigOptions = {
   cwd?: string,
@@ -140,6 +141,27 @@ export default class Config {
     return path.join(this.tempFolder, filename);
   }
 
+  /**
+   * Remote packages may be cached in a file system to be available for offline installation
+   * Second time the same package needs to be installed it will be loaded from there
+   */
+
+  getOfflineMirrorPath(registry: RegistryNames, tarUrl: ?string): string {
+    let mirrorPath = this.registries[registry] && this.registries[registry]
+      .config["kpm-offline-mirror"];
+    if (!mirrorPath) {
+      return "";
+    }
+    if (!tarUrl) {
+      return mirrorPath;
+    }
+    let parsed = url.parse(tarUrl);
+    if (!parsed || !parsed.pathname) {
+      return mirrorPath;
+    }
+    return path.join(mirrorPath, path.basename(parsed.pathname));
+  }
+
   /**
    * Find temporary folder.
    */
@@ -160,7 +182,7 @@ export default class Config {
       return opts.packagesRoot;
     }
 
-    // walk up from current directory looking for fbkpm_modules folders
+    // walk up from current directory looking for .fbkpm folders
     let parts = this.cwd.split(path.sep);
     for (let i = parts.length; i > 0; i--) {
       let loc = parts.slice(0, i).concat(constants.MODULE_CACHE_DIRECTORY).join(path.sep);

src/fetchers/_base.js

@@ -20,17 +20,19 @@ import * as fs from "../util/fs.js";
 let path = require("path");
 
 export default class BaseFetcher {
-  constructor(remote: PackageRemote, config: Config) {
-    this.reference = remote.reference;
-    this.registry  = remote.registry;
-    this.hash      = remote.hash;
-    this.config    = config;
+  constructor(remote: PackageRemote, config: Config, saveForOffline: boolean) {
+    this.reference      = remote.reference;
+    this.registry       = remote.registry;
+    this.hash           = remote.hash;
+    this.config         = config;
+    this.saveForOffline = saveForOffline;
   }
 
   registry: RegistryNames;
   reference: string;
   config: Config;
   hash: ?string;
+  saveForOffline: boolean;
 
   async _fetch(dest: string): Promise<string> {
     throw new Error("Not implemented");

src/fetchers/tarball.js

@@ -9,24 +9,61 @@
  * @flow
  */
 
-import { SecurityError } from "../errors.js";
+import { SecurityError, MessageError } from "../errors.js";
 import * as crypto from "../util/crypto.js";
 import BaseFetcher from "./_base.js";
+import * as fsUtil from "../util/fs.js";
 
 let zlib = require("zlib");
 let tar  = require("tar");
 let url  = require("url");
+let through = require("through2");
+let fs = require("fs");
+let path = require("path");
 
 export default class TarballFetcher extends BaseFetcher {
+
   async _fetch(dest: string): Promise<string> {
-    let { reference: ref, hash } = this;
+    let { reference: ref, hash, config, saveForOffline, registry } = this;
 
+    let parts = url.parse(ref);
     if (!hash) {
-      let parts = url.parse(ref);
       if (parts.protocol === "http:") {
         throw new SecurityError(`${ref}: Refusing to fetch tarball over plain HTTP without a hash`);
       }
     }
+    if (parts.protocol === null) {
+      let localTarball = path.resolve(
+        this.config.getOfflineMirrorPath(registry, null),
+        ref);
+      if (!await fsUtil.exists(localTarball)) {
+        throw new MessageError(`${ref}: Tarball is not in network and can't be located in cache`);
+      }
+      return new Promise((resolve, reject) => {
+        let validateStream = crypto.hashStreamValidation();
+
+        let extractor = tar.Extract({ path: dest, strip: 1 })
+          .on("error", reject)
+          .on("end", function () {
+            let expectHash = hash;
+            let actualHash = validateStream.getHash();
+            if (!expectHash || expectHash === actualHash) {
+              resolve(actualHash);
+            } else {
+              reject(new SecurityError(
+                `Bad hash. Expected ${expectHash} but got ${actualHash}`
+              ));
+            }
+          });
+        // flow gets confused with the pipe/on types chain
+        let cachedStream: Object = fs.createReadStream(localTarball);
+        cachedStream
+          .pipe(validateStream)
+          .pipe(zlib.createUnzip())
+          .on("error", reject)
+          .pipe(extractor);
+      });
+    }
 
     return this.config.requestManager.request({
       url: ref,
@@ -50,6 +87,19 @@ export default class TarballFetcher extends BaseFetcher {
             }
           });
 
+        let mirrorPath = config.getOfflineMirrorPath(registry, ref);
+        let mirrorTarballStream;
+        if (mirrorPath && saveForOffline) {
+          mirrorTarballStream = fs.createWriteStream(mirrorPath);
+          mirrorTarballStream.on("error", reject);
+        }
+        let mirrorSaver = through(function (chunk, enc, callback) {
+          if (mirrorTarballStream) {
+            mirrorTarballStream.write(chunk, enc);
+          }
+          callback(null, chunk);
+        });
+
         req
           .on("redirect", function () {
             if (hash) return;
@@ -64,6 +114,7 @@ export default class TarballFetcher extends BaseFetcher {
             }
           })
           .pipe(validateStream)
+          .pipe(mirrorSaver)
           .pipe(zlib.createUnzip())
           .on("error", reject)
           .pipe(extractor);

src/lockfile/index.js

@@ -24,13 +24,18 @@ export { default as parse } from "./parse";
 export { default as stringify } from "./stringify";
 
 export default class Lockfile {
-  constructor(cache: ?Object, strict?: boolean) {
+  constructor(cache: ?Object, strict?: boolean, save?: boolean) {
     this.strict = !!strict;
+    this.save = !!save;
     this.cache  = cache;
   }
 
+  // true if operation is just rehydrating node_modules folder
   strict: boolean;
 
+  // true if lockfile will be persisted
+  save: boolean;
+
   cache: ?{
     [key: string]: string | {
       name: string,
@@ -49,6 +54,7 @@ export default class Lockfile {
     dir: string,
     reporter: Reporter,
     strictIfPresent: boolean,
+    save: boolean,
   ): Promise<Lockfile> {
     // read the package.json in this directory
     let lockfileLoc = path.join(dir, constants.LOCKFILE_FILENAME);
@@ -68,7 +74,7 @@ export default class Lockfile {
       reporter.info("No lockfile found.");
     }
 
-    return new Lockfile(lockfile, strict);
+    return new Lockfile(lockfile, strict, save);
   }
 
   isStrict(): boolean {

src/package-fetcher.js

@@ -55,7 +55,7 @@ export default class PackageFetcher {
     await fs.mkdirp(dest);
 
     try {
-      let fetcher = new Fetcher(remote, this.config);
+      let fetcher = new Fetcher(remote, this.config, ref.saveForOffline);
       return await fetcher.fetch(dest);
     } catch (err) {
       try {

src/package-reference.js

@@ -22,6 +22,7 @@ export default class PackageReference {
     request: PackageRequest,
     info: Manifest,
     remote: PackageRemote,
+    saveForOffline: boolean,
   ) {
     this.lockfile = request.rootLockfile;
     this.requests = [request];
@@ -40,6 +41,7 @@ export default class PackageReference {
     this.patterns    = [];
     this.optional    = null;
     this.location    = null;
+    this.saveForOffline = !!saveForOffline;
   }
 
   requests: Array<PackageRequest>;
@@ -50,6 +52,7 @@ export default class PackageReference {
   version: string;
   uid: string;
   optional: ?boolean;
+  saveForOffline: boolean;
   dependencies: Array<string>;
   patterns: Array<string>;
   permissions: { [key: string]: boolean };

src/package-request.js

@@ -252,7 +252,7 @@ export default class PackageRequest {
     invariant(remote, "Missing remote");
 
     // set package reference
-    let ref = new PackageReference(this, info, remote);
+    let ref = new PackageReference(this, info, remote, this.resolver.lockfile.save);
 
     // in order to support lockfiles inside transitive dependencies we need to block
     // resolution to fetch the package so we can peek inside of it for a fbkpm.lock
@@ -270,9 +270,19 @@ export default class PackageRequest {
         info.name,
         () => this.resolver.fetcher.fetch(ref)
       );
+      let offlineMirrorPath = this.config.getOfflineMirrorPath(ref.remote.registry,
+        ref.remote.reference);
+      // replace resolved remote URL with local path
+      if (this.resolver.lockfile.save && offlineMirrorPath) {
+        if (await fs.exists(offlineMirrorPath)) {
+          remote.resolved = path.relative(
+            this.config.getOfflineMirrorPath(ref.remote.registry),
+            offlineMirrorPath) + `#${ref.remote.hash}`;
+        }
+      }
+      remote.hash = hash;
       newInfo.reference = ref;
       newInfo.remote = remote;
-      remote.hash = hash;
       info = newInfo;
 
       // find and load in fbkpm.lock from this module if it exists

src/registries/npm.js

@@ -15,6 +15,7 @@ import Registry from "./_base.js";
 let userHome = require("user-home");
 let path     = require("path");
 let _        = require("lodash");
+let ini      = require("ini");
 
 function getGlobalPrefix(): string {
   if (process.env.PREFIX) {
@@ -52,9 +53,13 @@ export default class NpmRegistry extends Registry {
 
     for (let loc of possibles) {
       if (await fs.exists(loc)) {
-        // TODO: merge it in!
+        _.defaults(this.config, ini.parse(await fs.readFile(loc)));
       }
     }
+    if (this.config["kpm-offline-mirror"]) {
+      this.config["kpm-offline-mirror"] = path.resolve(this.cwd, this.config["kpm-offline-mirror"]);
+      await fs.mkdirp(this.config["kpm-offline-mirror"]);
+    }
 
     _.defaults(this.config, {
       registry: "http://registry.npmjs.org"

src/resolvers/exotics/tarball.js

@@ -58,7 +58,7 @@ export default class TarballResolver extends ExoticResolver {
         reference: url,
         registry,
         hash
-      }, this.config);
+      }, this.config, false);
 
       // fetch file and get it's hash
       let fetched: FetchedManifest = await fetcher.fetch(dest);