From 375a055041b5ee49ca5fb3f74a58ca197c90c7d5 Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Tue, 27 Sep 2022 15:19:00 -0400 Subject: [PATCH] Emit lockfile v2 and fix bin links with NPM v7+ Lockfile v2 mostly just has a bit of extra metadata and all dependencies are hoisted to the top-level with path-specific keys in a new lock value called "packages". This update emits enough of the format that NPM v7+ seem to be happy enough with it and does not try to rewrite it and cause ENOTCACHED errors with the sandbox. As of NPM v7+, it no longer links bins for the top-level project automatically unless a global install is selected[1][2]. Given a global install would cause more problems than it would solve, I added a simple script to perform the linking ourselves and instructed `npm install` to never link them for consistency. Closes #236, #293, #294 [1]: https://github.com/npm/cli/commit/e46400c9484f5c66a0ba405eeb8c1340594dbf05#diff-24c01909dabbe2fc000fb5b43d14b511fb335b2f0c2e8e7a671f7d567a33d577R17-R18 [2]: https://github.com/npm/cli/issues/4308 --- bin/node2nix.js | 18 ++++++-- lib/Package.js | 2 +- nix/node-env.nix | 106 +++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 112 insertions(+), 14 deletions(-) diff --git a/bin/node2nix.js b/bin/node2nix.js index e4744b5..a04b6d5 100755 --- a/bin/node2nix.js +++ b/bin/node2nix.js @@ -28,7 +28,7 @@ var switches = [ ['-18', '--nodejs-18', 'Provides all settings to generate expression for usage with Node.js 18.x (default is: nodejs-14_x)'], ['--supplement-input FILE', 'A supplement package JSON file that are passed as build inputs to all packages defined in the input JSON file'], ['--supplement-output FILE', 'Path to a Nix expression representing a supplementing set of Nix packages provided as inputs to a project (defaults to: supplement.nix)'], - ['--include-peer-dependencies', 'Specifies whether to include peer dependencies. In npm 2.x, this is the default. (false by default)'], + ['--include-peer-dependencies', 'Specifies whether to include peer dependencies. In npm 2.x, this is the default. (true by default for Node.js 16+)'], ['--no-flatten', 'Simulate pre-npm 3.x isolated dependency structure. (false by default)'], ['--pkg-name NAME', 'Specifies the name of the Node.js package to use from Nixpkgs (defaults to: nodejs)'], ['--registry URL', 'URL referring to the NPM packages registry. It defaults to the official NPM one, but can be overridden to support private registries'], @@ -47,7 +47,7 @@ var parser = new optparse.OptionParser(switches); var help = false; var version = false; var production = true; -var includePeerDependencies = false; +var includePeerDependencies = true; var flatten = true; var inputJSON = "package.json"; var outputNix = "node-packages.nix"; @@ -118,61 +118,71 @@ parser.on('development', function(arg, value) { parser.on('nodejs-4', function(arg, value) { flatten = false; nodePackage = "nodejs-4_x"; - byPassCache = false; + bypassCache = false; + includePeerDependencies = false; }); parser.on('nodejs-6', function(arg, value) { flatten = true; nodePackage = "nodejs-6_x"; - byPassCache = false; + bypassCache = false; + includePeerDependencies = false; }); parser.on('nodejs-8', function(arg, value) { flatten = true; nodePackage = "nodejs-8_x"; bypassCache = true; + includePeerDependencies = false; }); parser.on('nodejs-10', function(arg, value) { flatten = true; nodePackage = "nodejs-10_x"; bypassCache = true; + includePeerDependencies = false; }); parser.on('nodejs-12', function(arg, value) { flatten = true; nodePackage = "nodejs-12_x"; bypassCache = true; + includePeerDependencies = false; }); parser.on('nodejs-13', function(arg, value) { flatten = true; nodePackage = "nodejs-13_x"; bypassCache = true; + includePeerDependencies = false; }); parser.on('nodejs-14', function(arg, value) { flatten = true; nodePackage = "nodejs-14_x"; bypassCache = true; + includePeerDependencies = false; }); parser.on('nodejs-16', function(arg, value) { flatten = true; nodePackage = "nodejs-16_x"; bypassCache = true; + includePeerDependencies = true; }); parser.on('nodejs-17', function(arg, value) { flatten = true; nodePackage = "nodejs-17_x"; bypassCache = true; + includePeerDependencies = true; }); parser.on('nodejs-18', function(arg, value) { flatten = true; nodePackage = "nodejs-18_x"; bypassCache = true; + includePeerDependencies = true; }); parser.on('include-peer-dependencies', function(arg, value) { diff --git a/lib/Package.js b/lib/Package.js index 9a6f0b7..9c4e21f 100644 --- a/lib/Package.js +++ b/lib/Package.js @@ -196,7 +196,7 @@ Package.prototype.resolveDependenciesAndSources = function(callback) { function(callback) { if(self.deploymentConfig.includePeerDependencies) { - /* Bundle the peer dependencies, if applicable */ + /* Bundle the required peer dependencies, if applicable */ self.bundleDependencies(resolvedDependencies, self.source.config.peerDependencies, callback); } else { callback(); diff --git a/nix/node-env.nix b/nix/node-env.nix index 2590dd2..5dad9ec 100644 --- a/nix/node-env.nix +++ b/nix/node-env.nix @@ -165,7 +165,11 @@ let if(process.argv[2] == "development") { replaceDependencies(packageObj.devDependencies); } + else { + packageObj.devDependencies = {}; + } replaceDependencies(packageObj.optionalDependencies); + replaceDependencies(packageObj.peerDependencies); /* Write the fixed package.json file */ fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); @@ -270,7 +274,7 @@ let # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes reconstructPackageLock = writeTextFile { - name = "addintegrityfields.js"; + name = "reconstructpackagelock.js"; text = '' var fs = require('fs'); var path = require('path'); @@ -280,25 +284,43 @@ let var lockObj = { name: packageObj.name, version: packageObj.version, - lockfileVersion: 1, + lockfileVersion: 2, requires: true, + packages: { + "": { + name: packageObj.name, + version: packageObj.version, + license: packageObj.license, + bin: packageObj.bin, + dependencies: packageObj.dependencies, + engines: packageObj.engines, + optionalDependencies: packageObj.optionalDependencies + } + }, dependencies: {} }; - function augmentPackageJSON(filePath, dependencies) { + function augmentPackageJSON(filePath, packages, dependencies) { var packageJSON = path.join(filePath, "package.json"); if(fs.existsSync(packageJSON)) { var packageObj = JSON.parse(fs.readFileSync(packageJSON)); + packages[filePath] = { + version: packageObj.version, + integrity: "sha1-000000000000000000000000000=", + dependencies: packageObj.dependencies, + engines: packageObj.engines, + optionalDependencies: packageObj.optionalDependencies + }; dependencies[packageObj.name] = { version: packageObj.version, integrity: "sha1-000000000000000000000000000=", dependencies: {} }; - processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies); + processDependencies(path.join(filePath, "node_modules"), packages, dependencies[packageObj.name].dependencies); } } - function processDependencies(dir, dependencies) { + function processDependencies(dir, packages, dependencies) { if(fs.existsSync(dir)) { var files = fs.readdirSync(dir); @@ -314,23 +336,84 @@ let pkgFiles.forEach(function(entry) { if(stats.isDirectory()) { var pkgFilePath = path.join(filePath, entry); - augmentPackageJSON(pkgFilePath, dependencies); + augmentPackageJSON(pkgFilePath, packages, dependencies); } }); } else { - augmentPackageJSON(filePath, dependencies); + augmentPackageJSON(filePath, packages, dependencies); } } }); } } - processDependencies("node_modules", lockObj.dependencies); + processDependencies("node_modules", lockObj.packages, lockObj.dependencies); fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); ''; }; + # Script that links bins defined in package.json to the node_modules bin directory + # NPM does not do this for top-level packages itself anymore as of v7 + linkBinsScript = writeTextFile { + name = "linkbins.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + var packageObj = JSON.parse(fs.readFileSync("package.json")); + + var nodeModules = Array(packageObj.name.split("/").length).fill("..").join(path.sep); + + if(packageObj.bin !== undefined) { + fs.mkdirSync(path.join(nodeModules, ".bin")) + + if(typeof packageObj.bin == "object") { + Object.keys(packageObj.bin).forEach(function(exe) { + if(fs.existsSync(packageObj.bin[exe])) { + console.log("linking bin '" + exe + "'"); + fs.symlinkSync( + path.join("..", packageObj.name, packageObj.bin[exe]), + path.join(nodeModules, ".bin", exe) + ); + } + else { + console.log("skipping non-existent bin '" + exe + "'"); + } + }) + } + else { + if(fs.existsSync(packageObj.bin)) { + console.log("linking bin '" + packageObj.bin + "'"); + fs.symlinkSync( + path.join("..", packageObj.name, packageObj.bin), + path.join(nodeModules, ".bin", packageObj.name.split("/").pop()) + ); + } + else { + console.log("skipping non-existent bin '" + packageObj.bin + "'"); + } + } + } + else if(packageObj.directories !== undefined && packageObj.directories.bin !== undefined) { + fs.mkdirSync(path.join(nodeModules, ".bin")) + + fs.readdirSync(packageObj.directories.bin).forEach(function(exe) { + if(fs.existsSync(path.join(packageObj.directories.bin, exe))) { + console.log("linking bin '" + exe + "'"); + fs.symlinkSync( + path.join("..", packageObj.name, packageObj.directories.bin, exe), + path.join(nodeModules, ".bin", exe) + ); + } + else { + console.log("skipping non-existent bin '" + exe + "'"); + } + }) + } + ''; + }; + prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: let forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; @@ -377,13 +460,18 @@ let npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild + runHook postRebuild + if [ "''${dontNpmInstall-}" != "1" ] then # NPM tries to download packages even when they already exist if npm-shrinkwrap is used. rm -f npm-shrinkwrap.json - npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install + npm ${forceOfflineFlag} --nodedir=${nodeSources} --no-bin-links --ignore-scripts ${npmFlags} ${lib.optionalString production "--production"} install fi + + # Link executables defined in package.json + node ${linkBinsScript} ''; # Builds and composes an NPM package including all its dependencies