diff --git a/gyp/pylib/gyp/easy_xml.py b/gyp/pylib/gyp/easy_xml.py index bda1a47468..41fb3c7c79 100644 --- a/gyp/pylib/gyp/easy_xml.py +++ b/gyp/pylib/gyp/easy_xml.py @@ -123,7 +123,10 @@ def WriteXmlIfChanged(content, path, encoding="utf-8", pretty=False, default_encoding = locale.getdefaultlocale()[1] if default_encoding and default_encoding.upper() != encoding.upper(): - xml_string = xml_string.encode(encoding) + if sys.platform == "win32": + if isinstance(xml_string, str): + xml_string = xml_string.decode("cp1251") # str --> bytes + xml_string = xml_string.encode(encoding) # bytes --> str # Get the old content try: diff --git a/lib/find-python-script.py b/lib/find-python-script.py new file mode 100644 index 0000000000..dae48e0e95 --- /dev/null +++ b/lib/find-python-script.py @@ -0,0 +1,4 @@ +import sys +if sys.stdout.encoding != "utf-8" and sys.platform == "win32": + sys.stdout.reconfigure(encoding='utf-8') +print(sys.executable) diff --git a/lib/find-python.js b/lib/find-python.js index a445e825b9..1c8262659f 100644 --- a/lib/find-python.js +++ b/lib/find-python.js @@ -1,5 +1,6 @@ 'use strict' +const path = require('path') const log = require('npmlog') const semver = require('semver') const cp = require('child_process') @@ -47,7 +48,7 @@ function PythonFinder (configPython, callback) { PythonFinder.prototype = { log: logWithPrefix(log, 'find Python'), - argsExecutable: ['-c', 'import sys; print(sys.executable);'], + argsExecutable: [path.resolve(__dirname, 'find-python-script.py')], argsVersion: ['-c', 'import sys; print("%s.%s.%s" % sys.version_info[:3]);'], semverRange: '>=3.6.0', @@ -274,7 +275,7 @@ PythonFinder.prototype = { run: function run (exec, args, shell, callback) { var env = extend({}, this.env) env.TERM = 'dumb' - const opts = { env: env, shell: shell } + const opts = { env: env, shell: shell, encoding: 'utf8' } this.log.silly('execFile: exec = %j', exec) this.log.silly('execFile: args = %j', args) diff --git a/test/rm.js b/test/rm.js new file mode 100644 index 0000000000..c32bfe0553 --- /dev/null +++ b/test/rm.js @@ -0,0 +1,20 @@ +const fs = require('fs') +const path = require('path') + +/** recursively delete files, symlinks (without following them) and dirs */ +module.exports = function rmRecSync (pth) { + pth = path.normalize(pth) + + rm(pth) + + function rm (pth) { + const pathStat = fs.statSync(pth) + // trick with lstat is used to avoid following symlinks (especially junctions on windows) + if (pathStat.isDirectory() && !fs.lstatSync(pth).isSymbolicLink()) { + fs.readdirSync(pth).forEach((nextPath) => rm(path.join(pth, nextPath))) + fs.rmdirSync(pth) + } else { + fs.unlinkSync(pth) + } + } +} diff --git a/test/test-find-python-script.js b/test/test-find-python-script.js new file mode 100644 index 0000000000..2b4376c36e --- /dev/null +++ b/test/test-find-python-script.js @@ -0,0 +1,84 @@ +// @ts-check +'use strict' +/** @typedef {import("tap")} Tap */ + +const test = require('tap').test +const execFile = require('child_process').execFile +const path = require('path') + +require('npmlog').level = 'warn' + +//* name can be used as short descriptions + +/** + * @typedef Check + * @property {string} path - path to executable or command + * @property {string} name - very little description + */ + +// TODO: add symlinks to python which will contain utf-8 chars +/** + * @type {Check[]} + */ +const checks = [ + { path: process.env.PYTHON, name: 'env var PYTHON' }, + { path: 'python3', name: 'python3 in PATH' }, + { path: 'python', name: 'python in PATH' } +] +const args = [path.resolve('./lib/find-python-script.py')] +const options = { + windowsHide: true +} + +/** + Getting output from find-python-script.py, + compare it to path provided to terminal. + If equals - test pass + + runs for all checks + + @private + @argument {Error} err - exec error + @argument {string} stdout - stdout buffer of child process + @argument {string} stderr + @this {{t: Tap, exec: Check}} + */ +function check (err, stdout, stderr) { + const { t, exec } = this + if (!err && !stderr) { + t.ok( + stdout.trim(), + `${exec.name}: check path ${exec.path} equals ${stdout.trim()}` + ) + } else { + // @ts-ignore + if (err.code === 9009 || err.code === 'ENOENT') { + t.skip(`skipped: ${exec.name} file not found`) + } else { + t.skip(`error: ${err}\n\nstderr: ${stderr}`) + } + } +} + +test('find-python-script', { buffered: false }, (t) => { + t.plan(checks.length) + + // ? may be more elegant way to pass context + // context for check functions + const ctx = { + t: t, + exec: {} + } + + for (const exec of checks) { + // checking if env var exist + if (!(exec.path === undefined || exec.path === null)) { + ctx.exec = exec + // passing ctx as copied object to make properties immutable from here + const boundedCheck = check.bind(Object.assign({}, ctx)) + execFile(exec.path, args, options, boundedCheck) + } else { + t.skip(`skipped: ${exec.name} doesn't exist or unavailable`) + } + } +}) diff --git a/test/test-find-python.js b/test/test-find-python.js index 67d0b2664f..6aa0579f19 100644 --- a/test/test-find-python.js +++ b/test/test-find-python.js @@ -1,226 +1,348 @@ -'use strict' - -delete process.env.PYTHON - -const test = require('tap').test -const findPython = require('../lib/find-python') -const execFile = require('child_process').execFile -const PythonFinder = findPython.test.PythonFinder - -require('npmlog').level = 'warn' - -test('find python', function (t) { - t.plan(4) - - findPython.test.findPython(null, function (err, found) { - t.strictEqual(err, null) - var proc = execFile(found, ['-V'], function (err, stdout, stderr) { - t.strictEqual(err, null) - t.ok(/Python 3/.test(stdout)) - t.strictEqual(stderr, '') - }) - proc.stdout.setEncoding('utf-8') - proc.stderr.setEncoding('utf-8') - }) -}) - -function poison (object, property) { - function fail () { - console.error(Error(`Property ${property} should not have been accessed.`)) - process.abort() - } - var descriptor = { - configurable: false, - enumerable: false, - get: fail, - set: fail - } - Object.defineProperty(object, property, descriptor) -} - -function TestPythonFinder () { - PythonFinder.apply(this, arguments) -} -TestPythonFinder.prototype = Object.create(PythonFinder.prototype) -// Silence npmlog - remove for debugging -TestPythonFinder.prototype.log = { - silly: () => {}, - verbose: () => {}, - info: () => {}, - warn: () => {}, - error: () => {} -} -delete TestPythonFinder.prototype.env.NODE_GYP_FORCE_PYTHON - -test('find python - python', function (t) { - t.plan(6) - - var f = new TestPythonFinder('python', done) - f.execFile = function (program, args, opts, cb) { - f.execFile = function (program, args, opts, cb) { - poison(f, 'execFile') - t.strictEqual(program, '/path/python') - t.ok(/sys\.version_info/.test(args[1])) - cb(null, '3.9.1') - } - t.strictEqual(program, - process.platform === 'win32' ? '"python"' : 'python') - t.ok(/sys\.executable/.test(args[1])) - cb(null, '/path/python') - } - f.findPython() - - function done (err, python) { - t.strictEqual(err, null) - t.strictEqual(python, '/path/python') - } -}) - -test('find python - python too old', function (t) { - t.plan(2) - - var f = new TestPythonFinder(null, done) - f.execFile = function (program, args, opts, cb) { - if (/sys\.executable/.test(args[args.length - 1])) { - cb(null, '/path/python') - } else if (/sys\.version_info/.test(args[args.length - 1])) { - cb(null, '2.3.4') - } else { - t.fail() - } - } - f.findPython() - - function done (err) { - t.ok(/Could not find any Python/.test(err)) - t.ok(/not supported/i.test(f.errorLog)) - } -}) - -test('find python - no python', function (t) { - t.plan(2) - - var f = new TestPythonFinder(null, done) - f.execFile = function (program, args, opts, cb) { - if (/sys\.executable/.test(args[args.length - 1])) { - cb(new Error('not found')) - } else if (/sys\.version_info/.test(args[args.length - 1])) { - cb(new Error('not a Python executable')) - } else { - t.fail() - } - } - f.findPython() - - function done (err) { - t.ok(/Could not find any Python/.test(err)) - t.ok(/not in PATH/.test(f.errorLog)) - } -}) - -test('find python - no python2, no python, unix', function (t) { - t.plan(2) - - var f = new TestPythonFinder(null, done) - f.checkPyLauncher = t.fail - f.win = false - - f.execFile = function (program, args, opts, cb) { - if (/sys\.executable/.test(args[args.length - 1])) { - cb(new Error('not found')) - } else { - t.fail() - } - } - f.findPython() - - function done (err) { - t.ok(/Could not find any Python/.test(err)) - t.ok(/not in PATH/.test(f.errorLog)) - } -}) - -test('find python - no python, use python launcher', function (t) { - t.plan(4) - - var f = new TestPythonFinder(null, done) - f.win = true - - f.execFile = function (program, args, opts, cb) { - if (program === 'py.exe') { - t.notEqual(args.indexOf('-3'), -1) - t.notEqual(args.indexOf('-c'), -1) - return cb(null, 'Z:\\snake.exe') - } - if (/sys\.executable/.test(args[args.length - 1])) { - cb(new Error('not found')) - } else if (f.winDefaultLocations.includes(program)) { - cb(new Error('not found')) - } else if (/sys\.version_info/.test(args[args.length - 1])) { - if (program === 'Z:\\snake.exe') { - cb(null, '3.9.0') - } else { - t.fail() - } - } else { - t.fail() - } - } - f.findPython() - - function done (err, python) { - t.strictEqual(err, null) - t.strictEqual(python, 'Z:\\snake.exe') - } -}) - -test('find python - no python, no python launcher, good guess', function (t) { - t.plan(2) - - var f = new TestPythonFinder(null, done) - f.win = true - const expectedProgram = f.winDefaultLocations[0] - - f.execFile = function (program, args, opts, cb) { - if (program === 'py.exe') { - return cb(new Error('not found')) - } - if (/sys\.executable/.test(args[args.length - 1])) { - cb(new Error('not found')) - } else if (program === expectedProgram && - /sys\.version_info/.test(args[args.length - 1])) { - cb(null, '3.7.3') - } else { - t.fail() - } - } - f.findPython() - - function done (err, python) { - t.strictEqual(err, null) - t.ok(python === expectedProgram) - } -}) - -test('find python - no python, no python launcher, bad guess', function (t) { - t.plan(2) - - var f = new TestPythonFinder(null, done) - f.win = true - - f.execFile = function (program, args, opts, cb) { - if (/sys\.executable/.test(args[args.length - 1])) { - cb(new Error('not found')) - } else if (/sys\.version_info/.test(args[args.length - 1])) { - cb(new Error('not a Python executable')) - } else { - t.fail() - } - } - f.findPython() - - function done (err) { - t.ok(/Could not find any Python/.test(err)) - t.ok(/not in PATH/.test(f.errorLog)) - } -}) +'use strict' + +const tap = require('tap') +const { test } = tap +const findPython = require('../lib/find-python') +const cp = require('child_process') +const PythonFinder = findPython.test.PythonFinder +const util = require('util') +const path = require('path') +const npmlog = require('npmlog') +const fs = require('fs') +npmlog.level = 'silent' + +// what final error message displayed in terminal should contain +const finalErrorMessage = 'Could not find any Python' + +//! don't forget manually call pythonFinderInstance.findPython() + +// String emulating path command or anything else with spaces +// and UTF-8 characters. +// Is returned by execFile +//! USE FOR ALL STRINGS +const testString = 'python one love♥' +const testVersions = { + outdated: '2.0.0', + normal: '3.9.0', + testError: new Error('test error') +} + +function strictDeepEqual (received, wanted) { + let result = false + + for (let i = 0; i < received.length; i++) { + if (Array.isArray(received[i]) && Array.isArray(wanted[i])) { + result = strictDeepEqual(received[i], wanted[i]) + } else { + result = received[i] === wanted[i] + } + + if (!result) { + return result + } + } + + return result +} + +/** + * @typedef OptionsObj + * @property {boolean} [shouldProduceError] pass test error to callback + * @property {boolean} [checkingPyLauncher] + * @property {boolean} [isPythonOutdated] return outdated version + * @property {boolean} [checkingWinDefaultPathes] + * + */ + +/** + * implement custom childProcess.execFile for testing proposes + * + * ! ***DO NOT FORGET TO OVERRIDE DEFAULT `PythonFinder.execFile` AFTER INSTANCING `PythonFinder`*** + * + * TODO: do overriding if automotive way + * + * @param {OptionsObj} [optionsObj] + */ +function TestExecFile (optionsObj) { + /** + * + * @this {PythonFinder} + */ + return function testExecFile (exec, args, options, callback) { + if (!(optionsObj && optionsObj.shouldProduceError)) { + // when checking version in checkExecPath, thus need to use PythonFinder.argsVersion + if (args === this.argsVersion) { + if (optionsObj && optionsObj.checkingWinDefaultPathes) { + if (this.winDefaultLocations.includes(exec)) { + callback(null, testVersions.normal) + } else { + callback(new Error('not found')) + } + } else if (optionsObj && optionsObj.isPythonOutdated) { + callback(null, testVersions.outdated, null) + } else { + callback(null, testVersions.normal, null) + } + } else if ( + // DONE: map through argsExecutable to check that all args are equals + strictDeepEqual(args, this.win ? this.argsExecutable.map((arg) => `"${arg}"`) : this.argsExecutable) + ) { + if (optionsObj && optionsObj.checkingPyLauncher) { + if ( + exec === 'py.exe' || + exec === (this.win ? '"python"' : 'python') + ) { + callback(null, testString, null) + } else { + callback(new Error('not found')) + } + } else if (optionsObj && optionsObj.checkingWinDefaultPathes) { + // return "not found" for regular checks (env-vars etc.) + // which are running twice: + // first to get path, second to check it + callback(new Error('not found')) + } else { + // returned string should be trimmed + callback(null, testString + '\n', null) + } + } else { + throw new Error( + `invalid arguments are provided! provided args +are: ${args};\n\nValid are: \n${this.argsExecutable}\n${this.argsVersion}` + ) + } + } else { + const testError = new Error( + `test error ${testString}; optionsObj: ${optionsObj}` + ) + callback(testError) + } + } +} + +/** + * + * @param {boolean} isPythonOutdated if true will return outdated version of python + * @param {OptionsObj} optionsObj + */ + +test('find-python', { buffered: true }, (t) => { + t.test('whole module tests', (t) => { + t.test('python found', (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + if (err) { + t.fail( + `mustn't produce any errors if execFile doesn't produced error. ${err}` + ) + } else { + t.equal(path, testString) + t.end() + } + }) + pythonFinderInstance.execFile = TestExecFile() + + pythonFinderInstance.findPython() + }) + + t.test('outdated version of python found', (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + if (!err) { + t.fail("mustn't return path for outdated version") + } else { + t.end() + } + }) + + pythonFinderInstance.execFile = TestExecFile({ isPythonOutdated: true }) + + pythonFinderInstance.findPython() + }) + + t.test('no python on computer', (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + t.ok(err.message.includes(finalErrorMessage)) + t.end() + }) + + pythonFinderInstance.execFile = TestExecFile({ + shouldProduceError: true + }) + + pythonFinderInstance.findPython() + }) + + t.test('no python, unix', (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + t.notOk(path) + + t.ok(err) + t.ok(err.message.includes(finalErrorMessage)) + t.end() + }) + + pythonFinderInstance.win = false + pythonFinderInstance.checkPyLauncher = t.fail + + pythonFinderInstance.execFile = TestExecFile({ + shouldProduceError: true + }) + + pythonFinderInstance.findPython() + }) + + t.test('no python, use python launcher', (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + t.equal(err, null) + + t.equal(path, testString) + + t.end() + }) + + pythonFinderInstance.win = true + + pythonFinderInstance.execFile = TestExecFile({ + checkingPyLauncher: true + }) + + pythonFinderInstance.findPython() + }) + + t.test( + 'no python, no python launcher, checking win default locations', + (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + t.equal(err, null) + t.ok(pythonFinderInstance.winDefaultLocations.includes(path)) + t.end() + }) + + pythonFinderInstance.win = true + + pythonFinderInstance.execFile = TestExecFile({ + checkingWinDefaultPathes: true + }) + pythonFinderInstance.findPython() + } + ) + + t.test('python is setted from config', (t) => { + const pythonFinderInstance = new PythonFinder(testString, (err, path) => { + t.equal(err, null) + + t.equal(path, testString) + + t.end() + }) + + pythonFinderInstance.win = true + + pythonFinderInstance.execFile = TestExecFile() + pythonFinderInstance.findPython() + }) + + t.end() + }) + + // DONE: make symlink to python with utf-8 chars + t.test('real testing', async (t) => { + const paths = { + python: '', + pythonDir: '', + testDir: '', + baseDir: __dirname + } + + const execFile = util.promisify(cp.execFile) + + // a bit tricky way to make PythonFinder promisified + function promisifyPythonFinder (config) { + let pythonFinderInstance + + const result = new Promise((resolve, reject) => { + pythonFinderInstance = new PythonFinder(config, (err, path) => { + if (err) { + reject(err) + } else { + resolve(path) + } + }) + }) + + return { pythonFinderInstance, result } + } + + async function testPythonPath (t, pythonPath) { + try { + const { stderr, stdout } = await execFile(pythonPath, ['-V']) + + console.log('stdout:', stdout) + console.log('stderr:', stderr) + + if (t.ok(stdout.includes('Python 3'), 'is it python with major version 3') && + t.equal(stderr, '', 'is stderr empty')) { + return true + } + + return false + } catch (err) { + t.equal(err, null, 'is error null') + return false + } + } + + // await is needed because test func is async + await t.test('trying to find real python exec', async (t) => { + const { pythonFinderInstance, result } = promisifyPythonFinder(null) + + try { + pythonFinderInstance.findPython() + + const pythonPath = await result + + if (t.ok(await testPythonPath(t, pythonPath), 'is path valid')) { + // stdout contain output of "python -V" command, not python path + // using found path as trusted + paths.python = pythonPath + paths.pythonDir = path.join(paths.python, '../') + } + } catch (err) { + t.notOk(err, 'are we having error') + } + + t.end() + }) + + await t.test(`test with path containing "${testString}"`, async (t) => { + // making fixture + paths.testDir = fs.mkdtempSync(path.resolve(paths.baseDir, 'node_modules', 'pythonFindTestFolder-')) + + // using "junction" to avoid permission error + fs.symlinkSync(paths.pythonDir, path.resolve(paths.testDir, testString), 'junction') + console.log('🚀 ~ file: test-find-python.js ~ line 312 ~ await.test ~ path.resolve(paths.testDir, testString)', path.resolve(paths.testDir, testString)) + console.log('🚀 ~ file: test-find-python.js ~ line 312 ~ await.test ~ paths.pythonDir', paths.pythonDir) + + const { pythonFinderInstance, result } = promisifyPythonFinder(path.resolve(paths.testDir, 'python')) + + pythonFinderInstance.findPython() + + const pythonPath = await result + + t.ok(await testPythonPath(t, pythonPath), 'is path valid') + + t.end() + }) + + // remove fixture + if (fs.rmSync) { + fs.rmSync(paths.testDir, { recursive: true }) + } else { + // + require('./rm.js')(paths.testDir) + } + + t.end() + }) + + t.end() +})