From 849929fd818955c5c8fd96e85ecaf8b656fc16c0 Mon Sep 17 00:00:00 2001 From: nebulon42 Date: Wed, 9 Mar 2016 19:18:07 +0100 Subject: [PATCH] refactor MML loading, support absolute paths in MML stylesheet file references --- README.md | 16 ++- bin/carto | 113 ++++------------ lib/carto/index.js | 1 + lib/carto/mml.js | 76 +++++++++++ test/errorhandling.test.js | 44 +++--- .../stylesheet_absolute_file.mml | 5 + .../stylesheet_absolute_file.result | 1 + .../stylesheet_absolute_file_syswin.result | 1 + test/rendering.test.js | 126 +++++++++--------- test/support/helper.js | 29 ++-- 10 files changed, 217 insertions(+), 195 deletions(-) create mode 100644 lib/carto/mml.js create mode 100644 test/errorhandling/stylesheet_absolute_file.mml create mode 100644 test/errorhandling/stylesheet_absolute_file.result create mode 100644 test/errorhandling/stylesheet_absolute_file_syswin.result diff --git a/README.md b/README.md index 3627d45d2..09c1bd99e 100644 --- a/README.md +++ b/README.md @@ -211,14 +211,19 @@ The `Renderer` interface is the main API for developers, and it takes an MML fil // defined variables: // - input (the name or identifier of the file being parsed) - // - data (a string containing the MML or an object of MML) var carto = require('carto'); try { - var output = new carto.Renderer({ - filename: input, - local_data_dir: path.dirname(input), - }).render(data); + var data = fs.readFileSync(input, 'utf-8'); + var mml = new carto.MML(); + mml.load(path.dirname(input), data, function (err, data) { + if (err) throw err; + var output = new carto.Renderer({ + filename: input, + local_data_dir: path.dirname(input), + }).render(data); + console.log(output); + }); } catch(err) { if (Array.isArray(err)) { err.forEach(function(e) { @@ -226,7 +231,6 @@ The `Renderer` interface is the main API for developers, and it takes an MML fil }); } else { throw err; } } - console.log(output); ### Vim diff --git a/bin/carto b/bin/carto index 70f9d6786..66470cb95 100755 --- a/bin/carto +++ b/bin/carto @@ -5,8 +5,7 @@ var path = require('path'), carto = require('../lib/carto'), semver = require('semver'), url = require('url'), - _ = require('lodash'), - yaml = require('js-yaml'); + _ = require('lodash'); var existsSync = require('fs').existsSync || require('path').existsSync @@ -65,12 +64,24 @@ if (!existsSync(input)) { process.exit(1); } -function compileMML(err, data) { - // force drain the millstone download pool now - // to ensure we can exit without waiting - if (options.localize && millstone.drainPool) { - millstone.drainPool(function() {}); - } +try { + var data = fs.readFileSync(input, 'utf-8'); +} catch(err) { + console.error("carto: " + err.message.replace(/^[A-Z]+, /, '')); + process.exit(1); +} + +if (ext === '.mml') { + var mml = new carto.MML(options); + mml.load(path.dirname(input), data, compile); +} else if (ext === '.mss') { + compile(null, data); +} else { + console.error("carto: please pass either a .mml file or .mss file"); +} + + +function compile(err, data) { if (err) { console.error(err); process.exit(1); @@ -84,7 +95,12 @@ function compileMML(err, data) { mapnik_version: options.api }); try { - var output = renderer.render(data); + var output; + if (ext === '.mml') { + output = renderer.render(data); + } else if (ext === '.mss') { + output = renderer.renderMSS(data); + } } catch (e) { if (e.stack) { console.error(e.stack); @@ -103,82 +119,3 @@ function compileMML(err, data) { console.log('TOTAL: ' + (duration) + 'ms'); } }; - -function compileMSS(err, data) { - if (err) { - console.error(err); - process.exit(1); - } - var renderer = new carto.Renderer({ - filename: path.basename(input), - benchmark: options.benchmark, - ppi: options.ppi - }, - { - mapnik_version: options.api - }); - try { - var output = renderer.renderMSS(data); - } catch (e) { - if (e.stack) { - console.error(e.stack); - } else { - console.error(e); - } - process.exit(1); - } - if (!options.benchmark) { - console.log(output); - } else { - var duration = (+new Date) - start; - console.log('TOTAL: ' + (duration) + 'ms'); - } -}; - -try { - var data = fs.readFileSync(input, 'utf-8'); -} catch(err) { - console.error("carto: " + err.message.replace(/^[A-Z]+, /, '')); - process.exit(1); -} - -if (ext == '.mml') { - try { - data = yaml.safeLoad(data); - } catch(err) { - console.error("carto: " + err.message.replace(/^[A-Z]+, /, '')); - process.exit(1); - } - - if (options.localize) { - var millstone = undefined; - try { - require.resolve('millstone'); - millstone = require('millstone'); - } catch (err) { - console.error('carto: Millstone not found, required if localizing stylesheet resources. ' + err.message.replace(/^[A-Z]+, /, '')); - process.exit(1); - } - millstone.resolve({ - mml: data, - base: path.dirname(input), - cache: path.join(path.dirname(input), 'cache'), - nosymlink: options.nosymlink - }, compileMML); - } else { - if (_.has(data, 'Stylesheet') && !_.isNil(data.Stylesheet)) { - data.Stylesheet = _.castArray(data.Stylesheet); - data.Stylesheet = data.Stylesheet.map(function(x) { - if (typeof x !== 'string') { - return { id: x, data: x.data } - } - return { id: x, data: fs.readFileSync(path.join(path.dirname(input), x), 'utf8') } - }); - } - compileMML(null,data); - } -} else if (ext == '.mss') { - compileMSS(null,data); -} else { - console.error("carto: please pass either a .mml file or .mss file"); -} diff --git a/lib/carto/index.js b/lib/carto/index.js index d10873173..0a1b91fbf 100644 --- a/lib/carto/index.js +++ b/lib/carto/index.js @@ -17,6 +17,7 @@ var carto = { version: getVersion(), Parser: require('./parser').Parser, Renderer: require('./renderer').Renderer, + MML: require('./mml').MML, tree: require('./tree'), // @TODO diff --git a/lib/carto/mml.js b/lib/carto/mml.js new file mode 100644 index 000000000..cfad6d718 --- /dev/null +++ b/lib/carto/mml.js @@ -0,0 +1,76 @@ +var path = require('path'), + fs = require('fs'), + _ = require('lodash'), + yaml = require('js-yaml'); + +var carto = require('./index'); + +carto.MML = function MML(options) { + this.options = options || {}; +}; + +/** + * Load a MML document. + * + * @param {String} basedir base directory of MML document. + * @param {String} data the MML document. + * @param {Callback} callback function to be called when finished loading. + */ +carto.MML.prototype.load = function load(basedir, data, callback) { + var mml = '', + that = this; + + try { + mml = yaml.safeLoad(data); + } catch (err) { + return callback("carto: " + err.message.replace(/^[A-Z]+, /, ''), null); + } + + if (this.options.localize) { + var millstone; + try { + require.resolve('millstone'); + millstone = require('millstone'); + } catch (err) { + return callback('carto: Millstone not found, required if localizing stylesheet resources. ' + err.message.replace(/^[A-Z]+, /, ''), null); + } + millstone.resolve({ + mml: mml, + base: basedir, + cache: path.join(basedir, 'cache'), + nosymlink: this.options.nosymlink + }, function (err, data) { + // force drain the millstone download pool now + // to ensure we can exit without waiting + if (that.options.localize && millstone.drainPool) { + millstone.drainPool(function() {}); + } + return callback(err, data); + }); + } else { + if (_.has(mml, 'Stylesheet') && !_.isNil(mml.Stylesheet)) { + mml.Stylesheet = _.castArray(mml.Stylesheet); + for (var i = 0; i < mml.Stylesheet.length; i++) { + var stylesheet = mml.Stylesheet[i]; + if (typeof stylesheet !== 'string') { + mml.Stylesheet[i] = stylesheet; + continue; + } + var mss, + file = path.resolve(basedir, stylesheet); + try { + mss = fs.readFileSync(file, 'utf-8'); + } catch (err) { + return callback('Failed to load file ' + file + ".\n", null); + } + mml.Stylesheet[i] = { id: stylesheet, data: mss }; + } + return callback(null, mml); + } + else { + return callback("Expecting a Stylesheet property containing an (array of) stylesheet object(s) of the form { id: 'x', 'data': 'y' }.\n", null); + } + } +}; + +module.exports = carto; diff --git a/test/errorhandling.test.js b/test/errorhandling.test.js index 603c51415..24da8c929 100644 --- a/test/errorhandling.test.js +++ b/test/errorhandling.test.js @@ -9,27 +9,29 @@ describe('Error handling mml+mss', function() { helper.files('errorhandling', 'mml', function(file) { var basename = path.basename(file); it('should handle errors in ' + basename, function(done) { - var mml = helper.mml(file); - try { - new carto.Renderer({ - paths: [ path.dirname(file) ], - data_dir: path.join(__dirname, '../data'), - local_data_dir: path.join(__dirname, 'rendering'), - filename: file - }).render(mml); - // should not get here - assert.ok(false); - done(); - } catch (err) { - if (err.message.indexOf('***') > -1) throw err; - var output = err.message; - // @TODO for some reason, fs.readFile includes an additional \n - // at the end of read files. Determine why. - // fs.writeFileSync(helper.resultFile(file), output); - var data = fs.readFileSync(helper.resultFile(file), 'utf8'); - assert.deepEqual(output, data); - done(); - } + helper.mml(file, function (err, mml) { + try { + if (err) throw new Error(err); + new carto.Renderer({ + paths: [ path.dirname(file) ], + data_dir: path.join(__dirname, '../data'), + local_data_dir: path.join(__dirname, 'rendering'), + filename: file + }).render(mml); + // should not get here + assert.ok(false); + done(); + } catch (err) { + if (err.message.indexOf('***') > -1) throw err; + var output = err.message; + // @TODO for some reason, fs.readFile includes an additional \n + // at the end of read files. Determine why. + // fs.writeFileSync(helper.resultFile(file), output); + var data = fs.readFileSync(helper.resultFile(file), 'utf8'); + assert.deepEqual(output, data); + done(); + } + }); }); }); }); diff --git a/test/errorhandling/stylesheet_absolute_file.mml b/test/errorhandling/stylesheet_absolute_file.mml new file mode 100644 index 000000000..a1fec3b65 --- /dev/null +++ b/test/errorhandling/stylesheet_absolute_file.mml @@ -0,0 +1,5 @@ +{ + "Stylesheet": [ + "/home/test/test.mss" + ] +} diff --git a/test/errorhandling/stylesheet_absolute_file.result b/test/errorhandling/stylesheet_absolute_file.result new file mode 100644 index 000000000..b4589131d --- /dev/null +++ b/test/errorhandling/stylesheet_absolute_file.result @@ -0,0 +1 @@ +Failed to load file /home/test/test.mss. diff --git a/test/errorhandling/stylesheet_absolute_file_syswin.result b/test/errorhandling/stylesheet_absolute_file_syswin.result new file mode 100644 index 000000000..ad4df5db3 --- /dev/null +++ b/test/errorhandling/stylesheet_absolute_file_syswin.result @@ -0,0 +1 @@ +Failed to load file C:\home\test\test.mss. diff --git a/test/rendering.test.js b/test/rendering.test.js index 129b10ea0..28b4be4b3 100644 --- a/test/rendering.test.js +++ b/test/rendering.test.js @@ -37,72 +37,74 @@ helper.files('rendering', 'mml', function(file) { } } it('should render ' + path.basename(file) + ' correctly', function(done) { - var mml = helper.mml(file); - try { - var env = { - paths: [ path.dirname(file) ], - data_dir: path.join(__dirname, '../data'), - local_data_dir: path.join(__dirname, 'rendering'), - filename: file - }, - renderer = null; + helper.mml(file, function (err, mml) { + try { + if (err) throw new Error(err); + var env = { + paths: [ path.dirname(file) ], + data_dir: path.join(__dirname, '../data'), + local_data_dir: path.join(__dirname, 'rendering'), + filename: file + }, + renderer = null; - if (api) { - renderer = new carto.Renderer(env, { - mapnik_version: api - }); - } - else { - renderer = new carto.Renderer(env); - } - var output = renderer.render(mml); - } catch (err) { - if (Array.isArray(err)){ - err.forEach(carto.writeError); - return done(); - } else { - return done(err); - } - } - var result = helper.resultFile(file); - helper.compareToXMLFile(result, output, function(err,expected_json,actual_json) { - var actual = file.replace(path.extname(file),'') + '-actual.json'; - var expected = file.replace(path.extname(file),'') + '-expected.json'; - if (err) { - // disabled since it can break on large diffs - /* - console.warn( - helper.stylize("Failure", 'red') + ': ' + - helper.stylize(file, 'underline') + - ' differs from expected result.'); - helper.showDifferences(err); - throw ''; - */ - fs.writeFileSync(actual,JSON.stringify(actual_json,null,4)); - fs.writeFileSync(expected,JSON.stringify(expected_json,null,4)); - throw new Error('failed: xml ' + result + ' in json form does not match expected result:\n' + actual + ' (actual)\n' + expected + ' (expected)'); - } else { - // cleanup any actual renders that no longer fail - try { - fs.unlinkSync(actual); - fs.unlinkSync(expected); - } catch (err) { - // do nothing + if (api) { + renderer = new carto.Renderer(env, { + mapnik_version: api + }); + } + else { + renderer = new carto.Renderer(env); + } + var output = renderer.render(mml); + } catch (err) { + if (Array.isArray(err)){ + err.forEach(carto.writeError); + return done(); + } else { + return done(err); } } - done(); - }, [ - helper.removeAbsoluteImages, - helper.removeAbsoluteDatasources - ]); + var result = helper.resultFile(file); + helper.compareToXMLFile(result, output, function(err,expected_json,actual_json) { + var actual = file.replace(path.extname(file),'') + '-actual.json'; + var expected = file.replace(path.extname(file),'') + '-expected.json'; + if (err) { + // disabled since it can break on large diffs + /* + console.warn( + helper.stylize("Failure", 'red') + ': ' + + helper.stylize(file, 'underline') + + ' differs from expected result.'); + helper.showDifferences(err); + throw ''; + */ + fs.writeFileSync(actual,JSON.stringify(actual_json,null,4)); + fs.writeFileSync(expected,JSON.stringify(expected_json,null,4)); + throw new Error('failed: xml ' + result + ' in json form does not match expected result:\n' + actual + ' (actual)\n' + expected + ' (expected)'); + } else { + // cleanup any actual renders that no longer fail + try { + fs.unlinkSync(actual); + fs.unlinkSync(expected); + } catch (err) { + // do nothing + } + } + done(); + }, [ + helper.removeAbsoluteImages, + helper.removeAbsoluteDatasources + ]); - // beforeExit(function() { - // if (!completed && renderResult) { - // console.warn(helper.stylize('renderer produced:', 'bold')); - // console.warn(renderResult); - // } - // assert.ok(completed, 'Rendering finished.'); - // }); + // beforeExit(function() { + // if (!completed && renderResult) { + // console.warn(helper.stylize('renderer produced:', 'bold')); + // console.warn(renderResult); + // } + // assert.ok(completed, 'Rendering finished.'); + // }); + }); }); }); }); diff --git a/test/support/helper.js b/test/support/helper.js index eeb80c51c..3cc0fe1a1 100644 --- a/test/support/helper.js +++ b/test/support/helper.js @@ -1,11 +1,10 @@ var path = require('path'), + carto = require('../../lib/carto'), fs = require('fs'), assert = require('assert'), crypto = require('crypto'), sax = require('sax'), diff = require('./diff').diff, - yaml = require('js-yaml'), - _ = require('lodash'), constants = ((!process.ENOENT) >= 1) ? require('constants') : { ENOENT: process.ENOENT }; @@ -36,22 +35,10 @@ exports.json = function(file, callback) { }); }; -exports.mml = function(file) { - var mml = yaml.safeLoad(fs.readFileSync(file, 'utf-8')); - if (!_.isNil(mml.Stylesheet)) { - mml.Stylesheet = mml.Stylesheet.map(function(s) { - if (path.extname(s) === '.mss') { - return { - id: s, - data: fs.readFileSync(path.join(path.dirname(file), s), 'utf-8') - }; - } - else { - return s; - } - }); - } - return mml; +exports.mml = function(file, callback) { + var data = fs.readFileSync(file, 'utf-8'); + var mml = new carto.MML(); + mml.load(path.dirname(file), data, callback); }; exports.mss = function(file) { @@ -122,6 +109,12 @@ exports.compareToXMLFile = function(filename, second, callback, processors) { }; exports.resultFile = function(file) { + if (process.platform && process.platform === 'win32') { + var fileWin = path.join(path.dirname(file), path.basename(file).replace(/\.\w+$/, '_syswin.result')); + if (fs.existsSync(fileWin)) { + return fileWin; + } + } return path.join(path.dirname(file), path.basename(file).replace(/\.\w+$/, '.result')); };