Add self hosting (#29)

* self-hosting

* add `dist/readme.md` placeholder

* new dist structure

* follow @sokra-senpai's advice

* bring back mjs to fix graphql

* improve build output

* bump version

.gitignore

@@ -1 +1,13 @@
-node_modules
+*
+dist/**/*.js
+!scripts
+!src
+!test
+!dist/
+!dist/ncc/
+!dist/buildin/
+!package.json
+!yarn.lock
+!.circleci
+!.*ignore
+!readme.md

.npmignore

@@ -1,5 +1,3 @@
 *
-!index.js
-!cli.js
-!webpack
-!webpack/node_modules
+!scripts/cli.js
+!dist/**/*.js

dist/buildin/readme.md

@@ -0,0 +1,7 @@
+# About this directory
+
+This directory will contain the webpack built-ins, like
+`module.js`, so that they can be accessed by webpack when
+it's being executeed inside the bundled ncc file.
+
+These files are published to npm.

dist/ncc/readme.md

@@ -0,0 +1,10 @@
+# About this directory
+
+This directory will contain:
+
+- `index.js` the main ncc bundle
+- `cli.js` the CLI bundle, excluding the main ncc bundle
+
+These are generated by the `build` step defined in `../../package.json`.
+
+These files are published to npm.

dist/readme.md

@@ -0,0 +1,9 @@
+# About this directory
+
+This directory will contain:
+- `buildin`: the webpack `buildin/` folder, required at runtime
+- `ncc`: the output from the ncc build
+
+They are generated by the `build` script defined in `../package.json`.
+
+This directory is the only one published to npm.

package.json

@@ -1,54 +1,32 @@
 {
   "name": "@zeit/ncc",
-  "version": "0.1.5-webpack",
-  "dependencies": {
-    "@webassemblyjs/ast": "1.7.11",
-    "@webassemblyjs/helper-module-context": "1.7.11",
-    "@webassemblyjs/wasm-edit": "1.7.11",
-    "@webassemblyjs/wasm-parser": "1.7.11",
-    "acorn": "^5.6.2",
-    "acorn-dynamic-import": "^3.0.0",
-    "ajv": "^6.1.0",
-    "ajv-keywords": "^3.1.0",
-    "arg": "^2.0.0",
-    "chrome-trace-event": "^1.0.0",
-    "enhanced-resolve": "^4.1.0",
-    "eslint-scope": "^4.0.0",
-    "json-parse-better-errors": "^1.0.2",
-    "loader-runner": "^2.3.0",
-    "loader-utils": "^1.1.0",
-    "memory-fs": "~0.4.1",
-    "micromatch": "^3.1.8",
-    "mkdirp": "~0.5.0",
-    "neo-async": "^2.5.0",
-    "node-libs-browser": "^2.0.0",
-    "resolve": "^1.8.1",
-    "schema-utils": "^0.4.4",
-    "tapable": "^1.1.0",
-    "uglifyjs-webpack-plugin": "^1.2.4",
-    "webpack-sources": "^1.3.0"
-  },
+  "version": "0.1.6",
   "license": "MIT",
-  "main": "index.js",
+  "main": "./dist/ncc/index.js",
   "bin": {
-    "ncc": "./cli.js"
+    "ncc": "./scripts/cli.js"
   },
   "scripts": {
-    "test": "jest",
+    "build": "node scripts/build",
+    "test": "npm run build && jest",
     "prepublish": "npm test"
   },
   "devDependencies": {
     "@google-cloud/datastore": "^2.0.0",
     "@sentry/node": "^4.3.0",
     "apollo-server-express": "^2.2.2",
+    "arg": "^2.0.0",
     "auth0": "^2.14.0",
     "aws-sdk": "^2.356.0",
     "axios": "^0.18.0",
+    "bytes": "^3.0.0",
+    "copy": "^0.3.2",
     "core-js": "^2.5.7",
     "cowsay": "^1.3.1",
     "express": "^4.16.4",
     "firebase": "^5.5.8",
     "fontkit": "^1.7.7",
+    "glob": "^7.1.3",
     "graphql": "^14.0.2",
     "ioredis": "^4.2.0",
     "jest": "^23.6.0",
@@ -65,35 +43,12 @@
     "react-dom": "^16.6.3",
     "redis": "^2.8.0",
     "request": "^2.88.0",
+    "resolve": "^1.8.1",
     "rxjs": "^6.3.3",
     "saslprep": "^1.0.2",
     "stripe": "^6.15.0",
     "the-answer": "^1.0.0",
-    "twilio": "^3.23.2"
-  },
-  "bundledDependencies": [
-    "@webassemblyjs/ast",
-    "@webassemblyjs/helper-module-context",
-    "@webassemblyjs/wasm-edit",
-    "@webassemblyjs/wasm-parser",
-    "acorn",
-    "acorn-dynamic-import",
-    "ajv",
-    "ajv-keywords",
-    "chrome-trace-event",
-    "enhanced-resolve",
-    "eslint-scope",
-    "json-parse-better-errors",
-    "loader-runner",
-    "loader-utils",
-    "memory-fs",
-    "micromatch",
-    "mkdirp",
-    "neo-async",
-    "node-libs-browser",
-    "schema-utils",
-    "tapable",
-    "uglifyjs-webpack-plugin",
-    "webpack-sources"
-  ]
+    "twilio": "^3.23.2",
+    "webpack": "^4.26.0"
+  }
 }

readme.md

@@ -51,7 +51,3 @@ require('@zeit/ncc')('/path/to/input', {
   console.log(bundle.code)
 })
 ```
-
-## TODO
-
-- [ ] Make self-hosting

scripts/build.js

@@ -0,0 +1,44 @@
+const ncc = require("../src/index.js");
+const { statSync, writeFileSync } = require("fs");
+const { promisify } = require("util");
+const { relative } = require("path");
+const copy = promisify(require("copy"));
+const glob = promisify(require("glob"));
+const bytes = require("bytes");
+
+async function main() {
+  const cli = await ncc(__dirname + "/../src/cli", {
+    externals: ["./index.js"]
+  });
+  const index = await ncc(__dirname + "/../src/index", {
+    // we dont care about watching, so we don't want
+    // to bundle it. even if we did want watching and a bigger
+    // bundle, webpack (and therefore ncc) cannot currently bundle
+    // chokidar, which is quite convenient
+    externals: ["chokidar"]
+  });
+
+  writeFileSync(__dirname + "/../dist/ncc/cli.js", cli);
+  writeFileSync(__dirname + "/../dist/ncc/index.js", index);
+
+  // copy webpack buildin
+  await copy(
+    __dirname + "/../node_modules/webpack/buildin/*.js",
+    __dirname + "/../dist/buildin/"
+  );
+
+  for (const file of await glob(__dirname + "/../dist/**/*.js")) {
+    console.log(
+      `✓ ${relative(__dirname + "/../", file)} (${bytes(
+        statSync(file).size
+      )})`
+    );
+  }
+}
+
+// remove me when node.js makes this the default behavior
+process.on("unhandledRejection", e => {
+  throw e;
+});
+
+main();

scripts/cli.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+// we separate `src/cli` and `scripts/cli` so that we can launch `ncc`
+// against a pure-js file without the shebang. in the future,
+// ncc should support taking in an input with a shebang,
+// and preserving it in the output
+require('../dist/ncc/cli');

cli.js → src/cli.js

@@ -1,8 +1,9 @@
-#!/usr/bin/env node
 const { resolve } = require("path");
 const args = require("arg")({
   "--help": Boolean,
   "-h": "--help",
+  "--external": [String],
+  "-e": "--external",
   "--out": String,
   "-o": "--out",
   "--no-minify": Boolean,
@@ -13,6 +14,7 @@ if (args["--help"]) {
   console.error(`ncc <input-file> [opts]
   Options:
     -M, --no-minify       Skip output minification
+    -e, --external [mod]  Skip bundling 'mod'. Can be used many times
     -o, --out [file]      Output file (defaults to stdout)
     -h, --help            Show help
 `);
@@ -25,8 +27,9 @@ if (args._.length !== 1) {
   process.exit(1);
 }
 
-const ncc = require("./")(resolve(args._[0]), {
-  minify: !args["--no-minify"]
+const ncc = require("./index.js")(resolve(args._[0]), {
+  minify: !args["--no-minify"],
+  externals: args["--external"]
 });
 
 ncc.then(code => {

index.js → src/index.js

@@ -1,28 +1,35 @@
 const resolve = require("resolve");
 const fs = require("fs");
-const webpack = require("./webpack/lib/webpack");
+const webpack = require("webpack");
 const MemoryFS = require("memory-fs");
 
-const SUPPORTED_EXTENSIONS = ['.js', '.json']
+const SUPPORTED_EXTENSIONS = [".mjs", ".js", ".json"];
 
-function resolveModule (context, request, callback) {
+function resolveModule(context, request, callback, forcedExternals = []) {
   const resolveOptions = {
     basedir: context,
     preserveSymlinks: true,
     extensions: SUPPORTED_EXTENSIONS
+  };
+
+  if (new Set(forcedExternals).has(request)) {
+    console.error(`ncc: Skipping bundling "${request}" per config`);
+    return callback(null, `commonjs ${request}`);
   }
 
-  resolve(request, resolveOptions, (err) => {
+  resolve(request, resolveOptions, err => {
     if (err) {
-      console.error(`Module directory "${context}" attempted to require "${request}" but could not be resolved, assuming external.`)
-      return callback(null, `commonjs ${request}`)
+      console.error(
+        `ncc: Module directory "${context}" attempted to require "${request}" but could not be resolved, assuming external.`
+      );
+      return callback(null, `commonjs ${request}`);
     }
 
-    callback()
-  })
+    callback();
+  });
 }
 
-module.exports = async (entry, { minify = true } = {}) => {
+module.exports = async (entry, { externals = [], minify = true } = {}) => {
   const mfs = new MemoryFS();
   const compiler = webpack({
     entry,
@@ -40,7 +47,9 @@ module.exports = async (entry, { minify = true } = {}) => {
     resolve: {
       extensions: SUPPORTED_EXTENSIONS
     },
-    externals: resolveModule,
+    // https://github.com/zeit/ncc/pull/29#pullrequestreview-177152175
+    node: false,
+    externals: (...args) => resolveModule(...[...args, externals]),
     plugins: [
       {
         apply(compiler) {