Skip to content

Commit

Permalink
Emit lockfile v2 and fix bin links with NPM v7+
Browse files Browse the repository at this point in the history 
Lockfile v2 mostly just has a bit of extra metadata and all dependencies
are hoisted to the top-level with path-specific keys in a new lock value
called "packages". This update emits enough of the format that NPM v7+
seem to be happy enough with it and does not try to rewrite it and cause
ENOTCACHED errors with the sandbox.

As of NPM v7+, it no longer links bins for the top-level project
automatically unless a global install is selected[1][2]. Given a global
install would cause more problems than it would solve, I added a simple
script to perform the linking ourselves and instructed `npm install` to
never link them for consistency.

Closes svanderburg#236, svanderburg#293, svanderburg#294

[1]: npm/cli@e46400c#diff-24c01909dabbe2fc000fb5b43d14b511fb335b2f0c2e8e7a671f7d567a33d577R17-R18
[2]: npm/cli#4308
  • Loading branch information
@lilyinstarlight
lilyinstarlight committed Sep 27, 2022
1 parent 0263600 commit f1ecc51
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 14 deletions.
18 changes: 14 additions & 4 deletions bin/node2nix.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ var switches = [
['-18', '--nodejs-18', 'Provides all settings to generate expression for usage with Node.js 18.x (default is: nodejs-14_x)'],
['--supplement-input FILE', 'A supplement package JSON file that are passed as build inputs to all packages defined in the input JSON file'],
['--supplement-output FILE', 'Path to a Nix expression representing a supplementing set of Nix packages provided as inputs to a project (defaults to: supplement.nix)'],
['--include-peer-dependencies', 'Specifies whether to include peer dependencies. In npm 2.x, this is the default. (false by default)'],
['--include-peer-dependencies', 'Specifies whether to include peer dependencies. In npm 2.x, this is the default. (true by default for Node.js 16+)'],
['--no-flatten', 'Simulate pre-npm 3.x isolated dependency structure. (false by default)'],
['--pkg-name NAME', 'Specifies the name of the Node.js package to use from Nixpkgs (defaults to: nodejs)'],
['--registry URL', 'URL referring to the NPM packages registry. It defaults to the official NPM one, but can be overridden to support private registries'],
Expand All @@ -47,7 +47,7 @@ var parser = new optparse.OptionParser(switches);
var help = false;
var version = false;
var production = true;
var includePeerDependencies = false;
var includePeerDependencies = true;
var flatten = true;
var inputJSON = "package.json";
var outputNix = "node-packages.nix";
Expand Down Expand Up @@ -118,61 +118,71 @@ parser.on('development', function(arg, value) {
parser.on('nodejs-4', function(arg, value) {
flatten = false;
nodePackage = "nodejs-4_x";
byPassCache = false;
bypassCache = false;
includePeerDependencies = false;
});

parser.on('nodejs-6', function(arg, value) {
flatten = true;
nodePackage = "nodejs-6_x";
byPassCache = false;
bypassCache = false;
includePeerDependencies = false;
});

parser.on('nodejs-8', function(arg, value) {
flatten = true;
nodePackage = "nodejs-8_x";
bypassCache = true;
includePeerDependencies = false;
});

parser.on('nodejs-10', function(arg, value) {
flatten = true;
nodePackage = "nodejs-10_x";
bypassCache = true;
includePeerDependencies = false;
});

parser.on('nodejs-12', function(arg, value) {
flatten = true;
nodePackage = "nodejs-12_x";
bypassCache = true;
includePeerDependencies = false;
});

parser.on('nodejs-13', function(arg, value) {
flatten = true;
nodePackage = "nodejs-13_x";
bypassCache = true;
includePeerDependencies = false;
});

parser.on('nodejs-14', function(arg, value) {
flatten = true;
nodePackage = "nodejs-14_x";
bypassCache = true;
includePeerDependencies = false;
});

parser.on('nodejs-16', function(arg, value) {
flatten = true;
nodePackage = "nodejs-16_x";
bypassCache = true;
includePeerDependencies = true;
});

parser.on('nodejs-17', function(arg, value) {
flatten = true;
nodePackage = "nodejs-17_x";
bypassCache = true;
includePeerDependencies = true;
});

parser.on('nodejs-18', function(arg, value) {
flatten = true;
nodePackage = "nodejs-18_x";
bypassCache = true;
includePeerDependencies = true;
});

parser.on('include-peer-dependencies', function(arg, value) {
Expand Down
2 changes: 1 addition & 1 deletion lib/Package.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ Package.prototype.resolveDependenciesAndSources = function(callback) {

function(callback) {
if(self.deploymentConfig.includePeerDependencies) {
/* Bundle the peer dependencies, if applicable */
/* Bundle the required peer dependencies, if applicable */
self.bundleDependencies(resolvedDependencies, self.source.config.peerDependencies, callback);
} else {
callback();
Expand Down
106 changes: 97 additions & 9 deletions nix/node-env.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,11 @@ let
if(process.argv[2] == "development") {
replaceDependencies(packageObj.devDependencies);
}
else {
packageObj.devDependencies = {};
}
replaceDependencies(packageObj.optionalDependencies);
replaceDependencies(packageObj.peerDependencies);
/* Write the fixed package.json file */
fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2));
Expand Down Expand Up @@ -270,7 +274,7 @@ let

# Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes
reconstructPackageLock = writeTextFile {
name = "addintegrityfields.js";
name = "reconstructpackagelock.js";
text = ''
var fs = require('fs');
var path = require('path');
Expand All @@ -280,25 +284,43 @@ let
var lockObj = {
name: packageObj.name,
version: packageObj.version,
lockfileVersion: 1,
lockfileVersion: 2,
requires: true,
packages: {
"": {
name: packageObj.name,
version: packageObj.version,
license: packageObj.license,
bin: packageObj.bin,
dependencies: packageObj.dependencies,
engines: packageObj.engines,
optionalDependencies: packageObj.optionalDependencies
}
},
dependencies: {}
};
function augmentPackageJSON(filePath, dependencies) {
function augmentPackageJSON(filePath, packages, dependencies) {
var packageJSON = path.join(filePath, "package.json");
if(fs.existsSync(packageJSON)) {
var packageObj = JSON.parse(fs.readFileSync(packageJSON));
packages[filePath] = {
version: packageObj.version,
integrity: "sha1-000000000000000000000000000=",
dependencies: packageObj.dependencies,
engines: packageObj.engines,
optionalDependencies: packageObj.optionalDependencies
};
dependencies[packageObj.name] = {
version: packageObj.version,
integrity: "sha1-000000000000000000000000000=",
dependencies: {}
};
processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies);
processDependencies(path.join(filePath, "node_modules"), packages, dependencies[packageObj.name].dependencies);
}
}
function processDependencies(dir, dependencies) {
function processDependencies(dir, packages, dependencies) {
if(fs.existsSync(dir)) {
var files = fs.readdirSync(dir);
Expand All @@ -314,23 +336,84 @@ let
pkgFiles.forEach(function(entry) {
if(stats.isDirectory()) {
var pkgFilePath = path.join(filePath, entry);
augmentPackageJSON(pkgFilePath, dependencies);
augmentPackageJSON(pkgFilePath, packages, dependencies);
}
});
} else {
augmentPackageJSON(filePath, dependencies);
augmentPackageJSON(filePath, packages, dependencies);
}
}
});
}
}
processDependencies("node_modules", lockObj.dependencies);
processDependencies("node_modules", lockObj.packages, lockObj.dependencies);
fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2));
'';
};

# Script that links bins defined in package.json to the node_modules bin directory
# NPM does not do this for top-level packages itself anymore as of v7
linkBinsScript = writeTextFile {
name = "linkbins.js";
text = ''
var fs = require('fs');
var path = require('path');
var packageObj = JSON.parse(fs.readFileSync("package.json"));
var nodeModules = Array(packageObj.name.split("/").length).fill("..").join(path.sep);
if(packageObj.bin !== undefined) {
fs.mkdirSync(path.join(nodeModules, ".bin"))
if(typeof packageObj.bin == "object") {
Object.keys(packageObj.bin).forEach(function(exe) {
if(fs.existsSync(packageObj.bin[exe])) {
console.log("linking bin '" + exe + "'");
fs.symlinkSync(
path.join("..", packageObj.name, packageObj.bin[exe]),
path.join(nodeModules, ".bin", exe)
);
}
else {
console.log("skipping non-executable bin '" + exe + "'");
}
})
}
else {
if(fs.existsSync(packageObj.bin)) {
console.log("linking bin '" + packageObj.bin + "'");
fs.symlinkSync(
path.join("..", packageObj.name, packageObj.bin),
path.join(nodeModules, ".bin", packageObj.name.split("/").pop())
);
}
else {
console.log("skipping non-executable bin '" + packageObj.bin + "'");
}
}
}
else if(packageObj.directories !== undefined && packageObj.directories.bin !== undefined) {
fs.mkdirSync(path.join(nodeModules, ".bin"))
fs.readdirSync(packageObj.directories.bin).forEach(function(exe) {
if(fs.existsSync(path.join(packageObj.directories.bin, exe))) {
console.log("linking bin '" + exe + "'");
fs.symlinkSync(
path.join("..", packageObj.name, packageObj.directories.bin, exe),
path.join(nodeModules, ".bin", exe)
);
}
else {
console.log("skipping non-executable bin '" + exe + "'");
}
})
}
'';
};

prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}:
let
forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
Expand Down Expand Up @@ -377,13 +460,18 @@ let
npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild
runHook postRebuild
if [ "''${dontNpmInstall-}" != "1" ]
then
# NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
rm -f npm-shrinkwrap.json
npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install
npm ${forceOfflineFlag} --nodedir=${nodeSources} --no-bin-links --ignore-scripts ${npmFlags} ${lib.optionalString production "--production"} install
fi
# Link executables defined in package.json
node ${linkBinsScript}
'';

# Builds and composes an NPM package including all its dependencies
Expand Down

0 comments on commit f1ecc51

Please sign in to comment.