From bdc88d97ac3849e9d3659439c86279f2a5433c13 Mon Sep 17 00:00:00 2001 From: Etienne Rossignon Date: Thu, 22 Apr 2021 18:00:05 +0200 Subject: [PATCH] fix .node loading issue on windows (#1135) --- .eslintignore | 1 + prelude/bootstrap.js | 101 +++++++++++++++++++++--------- test/test-1135-issue/.gitignore | 2 + test/test-1135-issue/index.js | 5 ++ test/test-1135-issue/main.js | 67 ++++++++++++++++++++ test/test-1135-issue/package.json | 22 +++++++ 6 files changed, 167 insertions(+), 31 deletions(-) create mode 100644 test/test-1135-issue/.gitignore create mode 100644 test/test-1135-issue/index.js create mode 100644 test/test-1135-issue/main.js create mode 100755 test/test-1135-issue/package.json diff --git a/.eslintignore b/.eslintignore index 038121cc6..9e901f174 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,2 +1,3 @@ lib-es5 node_modules +dist diff --git a/prelude/bootstrap.js b/prelude/bootstrap.js index 676d7a45c..8cc73153e 100644 --- a/prelude/bootstrap.js +++ b/prelude/bootstrap.js @@ -1757,6 +1757,8 @@ function payloadFileSync(pointer) { // ///////////////////////////////////////////////////////////////// (() => { const fs = require('fs'); + const path = require('path'); + var ancestor = {}; ancestor.dlopen = process.dlopen; @@ -1768,28 +1770,12 @@ function payloadFileSync(pointer) { process.dlopen = function dlopen() { const args = cloneArgs(arguments); const modulePath = revertMakingLong(args[1]); - const moduleDirname = require('path').dirname(modulePath); - if (insideSnapshot(modulePath)) { - // Node addon files and .so cannot be read with fs directly, they are loaded with process.dlopen which needs a filesystem path - // we need to write the file somewhere on disk first and then load it - const moduleContent = fs.readFileSync(modulePath); - const moduleBaseName = require('path').basename(modulePath); - const hash = require('crypto') - .createHash('sha256') - .update(moduleContent) - .digest('hex'); - const tmpModulePath = `${require('os').tmpdir()}/${hash}_${moduleBaseName}`; - try { - fs.statSync(tmpModulePath); - } catch (e) { - // Most likely this means the module is not on disk yet - fs.writeFileSync(tmpModulePath, moduleContent, { mode: 0o444 }); - } - args[1] = tmpModulePath; - } - + const moduleDirname = path.dirname(modulePath); + const moduleBaseName = path.basename(modulePath); + const moduleFolder = path.dirname(modulePath); const unknownModuleErrorRegex = /([^:]+): cannot open shared object file: No such file or directory/; - const tryImporting = function tryImporting(previousErrorMessage) { + + function tryImporting(_tmpFolder, previousErrorMessage) { try { const res = ancestor.dlopen.apply(process, args); return res; @@ -1799,24 +1785,77 @@ function payloadFileSync(pointer) { throw e; } if (e.message.match(unknownModuleErrorRegex)) { + // this case triggers on linux, the error message give us a clue on what dynamic linking library + // is missing. // some modules are packaged with dynamic linking and needs to open other files that should be in // the same directory, in this case, we write this file in the same /tmp directory and try to // import the module again + const moduleName = e.message.match(unknownModuleErrorRegex)[1]; - const importModulePath = `${moduleDirname}/${moduleName}`; - const moduleContent = fs.readFileSync(importModulePath); - const moduleBaseName = require('path').basename(importModulePath); - const tmpModulePath = `${require('os').tmpdir()}/${moduleBaseName}`; + const importModulePath = path.join(moduleDirname, moduleName); + + if (!fs.existsSync(importModulePath)) { + throw new Error( + "INTERNAL ERRROR this files does'nt exist in the virtual file system", + importModulePath + ); + } + const moduleContent1 = fs.readFileSync(importModulePath); + const tmpModulePath1 = path.join(_tmpFolder, moduleName); + + console.log(' Cannot read :', e.message); + try { - fs.statSync(tmpModulePath); + fs.statSync(tmpModulePath1); } catch (err) { - fs.writeFileSync(tmpModulePath, moduleContent, { mode: 0o444 }); + fs.writeFileSync(tmpModulePath1, moduleContent1, { mode: 0o444 }); } - return tryImporting(e.message); + return tryImporting(_tmpFolder, e.message); + } else { + // this case triggers on windows mainly. + // we copy all stuff that exists in the folder of the .node module + // into the tempory folders... + const f = fs.readdirSync(moduleFolder); + for (const g of f) { + if (g === moduleBaseName) { + // ignore the current module + continue; + } + const filenameSrc = path.join(moduleFolder, g); + const filenameDst = path.join(_tmpFolder, g); + const content = fs.readFileSync(filenameSrc); + fs.writeFileSync(filenameDst, content, { mode: 0o444 }); + } + return tryImporting(_tmpFolder, e.message); } - throw e; } - }; - tryImporting(); + } + if (insideSnapshot(modulePath)) { + const moduleContent = fs.readFileSync(modulePath); + + // Node addon files and .so cannot be read with fs directly, they are loaded with process.dlopen which needs a filesystem path + // we need to write the file somewhere on disk first and then load it + const hash = require('crypto') + .createHash('sha256') + .update(moduleContent) + .digest('hex'); + + const _tmpFolder = path.join(require('os').tmpdir(), hash); + if (!fs.existsSync(_tmpFolder)) { + fs.mkdirSync(_tmpFolder); + } + const tmpModulePath = path.join(_tmpFolder, moduleBaseName); + + try { + fs.statSync(tmpModulePath); + } catch (e) { + // Most likely this means the module is not on disk yet + fs.writeFileSync(tmpModulePath, moduleContent, { mode: 0o777 }); + } + args[1] = tmpModulePath; + tryImporting(_tmpFolder); + } else { + return ancestor.dlopen.apply(process, args); + } }; })(); diff --git a/test/test-1135-issue/.gitignore b/test/test-1135-issue/.gitignore new file mode 100644 index 000000000..7a7b33aa1 --- /dev/null +++ b/test/test-1135-issue/.gitignore @@ -0,0 +1,2 @@ +package-lock.json +node_modules diff --git a/test/test-1135-issue/index.js b/test/test-1135-issue/index.js new file mode 100644 index 000000000..60d13162d --- /dev/null +++ b/test/test-1135-issue/index.js @@ -0,0 +1,5 @@ +'use strict'; + +/* eslint-disable no-unused-vars */ +var canvas = require('canvas'); +console.log('42'); diff --git a/test/test-1135-issue/main.js b/test/test-1135-issue/main.js new file mode 100644 index 000000000..ddb70ea3e --- /dev/null +++ b/test/test-1135-issue/main.js @@ -0,0 +1,67 @@ +#!/usr/bin/env node + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const assert = require('assert'); +const utils = require('../utils.js'); + +assert(__dirname === process.cwd()); + +const target = process.argv[2] || 'host'; +const input = './package.json'; +const output = './test-output.exe'; + +console.log('target = ', target); + +// remove any possible left-over +utils.vacuum.sync('./node_modules'); + +// npm instal -g node-pre-gyp +utils.exec.sync('npm install -g node-pre-gyp'); + +// launch `npm install` +const npmlog = utils.spawn.sync( + path.join( + path.dirname(process.argv[0]), + 'npx' + (process.platform === 'win32' ? '.cmd' : '') + ), + ['npm', 'install'], + { cwd: path.dirname(output), expect: 0 } +); +console.log('npm log :', npmlog); + +// verify that we have the .pnpm folder and a symlinks module in node_modules +assert(fs.lstatSync(path.join(__dirname, 'node_modules/canvas')).isDirectory()); +assert( + fs.lstatSync(path.join(__dirname, 'node_modules/canvas/build')).isDirectory() +); +assert( + fs + .lstatSync(path.join(__dirname, 'node_modules/canvas/build/Release')) + .isDirectory() +); +assert( + fs + .lstatSync( + path.join(__dirname, 'node_modules/canvas/build/Release/canvas.node') + ) + .isFile() +); + +utils.pkg.sync(['--target', target, '--debug', '--output', output, input]); + +// check that produced executable is running and produce the expected output. +const log = utils.spawn.sync(output, [], { + cwd: path.dirname(output), + expect: 0, +}); +assert(log === '42\n'); + +// clean up +utils.vacuum.sync(output); +utils.vacuum.sync('./node_modules'); +utils.vacuum.sync('./package-lock.json'); + +console.log('OK'); diff --git a/test/test-1135-issue/package.json b/test/test-1135-issue/package.json new file mode 100755 index 000000000..60b8224f1 --- /dev/null +++ b/test/test-1135-issue/package.json @@ -0,0 +1,22 @@ +{ + "name": "pkg_issue_1335", + "version": "1.0.0", + "description": "", + "main": "index.js", + "bin": "./index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "pkg42": "npx pkg -t host package.json" + }, + "pkg": { + "scripts": "index.js", + "assets": "node_modules/canvas/build/Release/*", + "outputPath": "dist2" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "canvas": "^2.7.0" + } +}