diff --git a/.codeclimate.yml b/.codeclimate.yml index a23fa86..dfe3bb6 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -1,7 +1,7 @@ engines: eslint: enabled: true - channel: "eslint-4" + channel: "eslint-6" config: config: ".eslintrc.yaml" diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml index 57d3f7c..66e92fc 100644 --- a/.github/workflows/coveralls.yml +++ b/.github/workflows/coveralls.yml @@ -15,10 +15,10 @@ jobs: with: fetch-depth: 1 - - name: Use Node.js 10 + - name: Use Node.js 12 uses: actions/setup-node@master with: - version: 10.x + version: 12.x - name: install, run run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8a0b119..03651fa 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,4 +30,5 @@ jobs: run: npm run lint env: - CI: true \ No newline at end of file + CI: true + diff --git a/Changes.md b/Changes.md index dcf39bd..21458d5 100644 --- a/Changes.md +++ b/Changes.md @@ -1,6 +1,12 @@ ## 1.N.NN - 20YY-MM-DD +## 1.0.19 - 2019-MM-DD + +- configfile: convert to es6 class +- configfile.read_dir: promisify +- configfile: use simpler es6 `for..in` and `for..of` + ## 1.0.18 - 2019-10-11 diff --git a/config.js b/config.js index 2fc0a71..41d1dbd 100644 --- a/config.js +++ b/config.js @@ -52,7 +52,9 @@ class Config { } getDir (name, opts, done) { - cfreader.read_dir(path.resolve(this.root_path, name), opts, done); + cfreader.read_dir(path.resolve(this.root_path, name), opts).then((files) => { + done(null, files) // keep the API consistent + }).catch(done) } arrange_args (args) { diff --git a/configfile.js b/configfile.js index 7c708a7..6b2f347 100644 --- a/configfile.js +++ b/configfile.js @@ -1,404 +1,394 @@ 'use strict'; -// Config file loader const fs = require('fs'); const path = require('path'); -// for "ini" type files -const regex = exports.regex = { - section: /^\s*\[\s*([^\]]*?)\s*\]\s*$/, - param: /^\s*([\w@:._\-/[\]]+)\s*(?:=\s*(.*?)\s*)?$/, - comment: /^\s*[;#].*$/, - line: /^\s*(.*?)\s*$/, - blank: /^\s*$/, - continuation: /\\[ \t]*$/, - is_integer: /^-?\d+$/, - is_float: /^-?\d+\.\d+$/, - is_truth: /^(?:true|yes|ok|enabled|on|1)$/i, - is_array: /(.+)\[\]$/, -}; - -const cfreader = exports; - -cfreader.watch_files = true; -cfreader._config_cache = {}; -cfreader._read_args = {}; -cfreader._watchers = {}; -cfreader._enoent_timer = false; -cfreader._enoent_files = {}; -cfreader._sedation_timers = {}; -cfreader._overrides = {}; - let config_dir_candidates = [ // these work when this file is loaded as require('./config.js') path.join(__dirname, 'config'), // Haraka ./config dir __dirname, // npm packaged plugins ]; -cfreader.get_path_to_config_dir = () => { - if (process.env.HARAKA) { - // console.log(`process.env.HARAKA: ${process.env.HARAKA}`); - cfreader.config_path = path.join(process.env.HARAKA, 'config'); - return; +class cfreader { + constructor () { + this.watch_files = true; + this._config_cache = {}; + this._read_args = {}; + this._watchers = {}; + this._enoent_timer = false; + this._enoent_files = {}; + this._sedation_timers = {}; + this._overrides = {}; + + this.get_path_to_config_dir() + + // for "ini" type files + this.regex = { + section: /^\s*\[\s*([^\]]*?)\s*\]\s*$/, + param: /^\s*([\w@:._\-/[\]]+)\s*(?:=\s*(.*?)\s*)?$/, + comment: /^\s*[;#].*$/, + line: /^\s*(.*?)\s*$/, + blank: /^\s*$/, + continuation: /\\[ \t]*$/, + is_integer: /^-?\d+$/, + is_float: /^-?\d+\.\d+$/, + is_truth: /^(?:true|yes|ok|enabled|on|1)$/i, + is_array: /(.+)\[\]$/, + } } - if (process.env.NODE_ENV === 'test') { - // loaded by haraka-config/test/* - cfreader.config_path = path.join(__dirname, 'test', 'config'); - return; + get_path_to_config_dir () { + if (process.env.HARAKA) { + // console.log(`process.env.HARAKA: ${process.env.HARAKA}`); + this.config_path = path.join(process.env.HARAKA, 'config'); + return; + } + + if (process.env.NODE_ENV === 'test') { + // loaded by haraka-config/test/* + this.config_path = path.join(__dirname, 'test', 'config'); + return; + } + + // these work when this is loaded with require('haraka-config') + if (/node_modules[\\/]haraka-config$/.test(__dirname)) { + config_dir_candidates = [ + path.join(__dirname, '..', '..', 'config'), // haraka/Haraka/* + path.join(__dirname, '..', '..'), // npm packaged modules + ] + } + + for (const candidate of config_dir_candidates) { + try { + const stat = fs.statSync(candidate); + if (stat && stat.isDirectory()) { + this.config_path = candidate; + return; + } + } + catch (ignore) { + console.error(ignore.message); + } + } } - // these work when this is loaded with require('haraka-config') - if (/node_modules[\\/]haraka-config$/.test(__dirname)) { - config_dir_candidates = [ - path.join(__dirname, '..', '..', 'config'), // haraka/Haraka/* - path.join(__dirname, '..', '..'), // npm packaged modules - ] + on_watch_event (name, type, options, cb) { + return (fse, filename) => { + if (this._sedation_timers[name]) { + clearTimeout(this._sedation_timers[name]); + } + this._sedation_timers[name] = setTimeout(() => { + console.log(`Reloading file: ${name}`); + this.load_config(name, type, options); + delete this._sedation_timers[name]; + if (typeof cb === 'function') cb(); + }, 5 * 1000); + + if (fse !== 'rename') return; + // https://github.com/joyent/node/issues/2062 + // After a rename event, re-watch the file + this._watchers[name].close(); + try { + this._watchers[name] = fs.watch(name, { persistent: false }, this.on_watch_event(...arguments)); + } + catch (e) { + if (e.code === 'ENOENT') { + this._enoent_files[name] = true; + this.ensure_enoent_timer(); + } + else { + console.error(`Error watching file: ${name} : ${e}`); + } + } + } } - for (let i=0; i < config_dir_candidates.length; i++) { - const candidate = config_dir_candidates[i]; + watch_dir () { + // NOTE: Has OS platform limitations: + // https://nodejs.org/api/fs.html#fs_fs_watch_filename_options_listener + const cp = this.config_path; + if (this._watchers[cp]) return; + try { - const stat = fs.statSync(candidate); - if (stat && stat.isDirectory()) { - cfreader.config_path = candidate; - return; - } + this._watchers[cp] = fs.watch(cp, { persistent: false }, (fse, filename) => { + if (!filename) return; + const full_path = path.join(cp, filename); + if (!this._read_args[full_path]) return; + const args = this._read_args[full_path]; + if (args.options && args.options.no_watch) return; + if (this._sedation_timers[filename]) { + clearTimeout(this._sedation_timers[filename]); + } + this._sedation_timers[filename] = setTimeout(() => { + console.log(`Reloading file: ${full_path}`); + this.load_config(full_path, args.type, args.options); + delete this._sedation_timers[filename]; + if (typeof args.cb === 'function') args.cb(); + }, 5 * 1000); + }); } - catch (ignore) { - console.error(ignore.message); + catch (e) { + console.error(`Error watching directory ${cp}(${e})`); } + return; } -} -exports.get_path_to_config_dir(); -// console.log(`cfreader.config_path: ${cfreader.config_path}`); -cfreader.on_watch_event = (name, type, options, cb) => { - return function (fse, filename) { - if (cfreader._sedation_timers[name]) { - clearTimeout(cfreader._sedation_timers[name]); - } - cfreader._sedation_timers[name] = setTimeout(() => { - console.log(`Reloading file: ${name}`); - cfreader.load_config(name, type, options); - delete cfreader._sedation_timers[name]; - if (typeof cb === 'function') cb(); - }, 5 * 1000); - - if (fse !== 'rename') return; - // https://github.com/joyent/node/issues/2062 - // After a rename event, re-watch the file - cfreader._watchers[name].close(); + watch_file (name, type, cb, options) { + // This works on all OS's, but watch_dir() above is preferred for Linux and + // Windows as it is far more efficient. + // NOTE: we need a fs.watch per file. It's impossible to watch non-existent + // files. Instead, note which files we attempted + // to watch that returned ENOENT and fs.stat each periodically + if (this._watchers[name] || (options && options.no_watch)) return; + try { - cfreader._watchers[name] = fs.watch( - name, - { persistent: false }, - cfreader.on_watch_event(name, type, options, cb)); + this._watchers[name] = fs.watch(name, {persistent: false}, this.on_watch_event(name, type, options, cb)); } catch (e) { - if (e.code === 'ENOENT') { - cfreader._enoent_files[name] = true; - cfreader.ensure_enoent_timer(); + if (e.code !== 'ENOENT') { // ignore error when ENOENT + console.error(`Error watching config file: ${name} : ${e}`); } else { - console.error(`Error watching file: ${name} : ${e}`); + this._enoent_files[name] = true; + this.ensure_enoent_timer(); } } - }; -} - -cfreader.watch_dir = () => { - // NOTE: Has OS platform limitations: - // https://nodejs.org/api/fs.html#fs_fs_watch_filename_options_listener - const cp = cfreader.config_path; - if (cfreader._watchers[cp]) return; - - function watcher (fse, filename) { - if (!filename) return; - const full_path = path.join(cp, filename); - if (!cfreader._read_args[full_path]) return; - const args = cfreader._read_args[full_path]; - if (args.options && args.options.no_watch) return; - if (cfreader._sedation_timers[filename]) { - clearTimeout(cfreader._sedation_timers[filename]); - } - cfreader._sedation_timers[filename] = setTimeout(() => { - console.log(`Reloading file: ${full_path}`); - cfreader.load_config(full_path, args.type, args.options); - delete cfreader._sedation_timers[filename]; - if (typeof args.cb === 'function') args.cb(); - }, 5 * 1000); - } - try { - cfreader._watchers[cp] = fs.watch(cp, { persistent: false }, watcher); - } - catch (e) { - console.error(`Error watching directory ${cp}(${e})`); - } - return; -} -cfreader.watch_file = (name, type, cb, options) => { - // This works on all OS's, but watch_dir() above is preferred for Linux and - // Windows as it is far more efficient. - // NOTE: we need a fs.watch per file. It's impossible to watch non-existent - // files. Instead, note which files we attempted - // to watch that returned ENOENT and fs.stat each periodically - if (cfreader._watchers[name] || (options && options.no_watch)) return; - try { - cfreader._watchers[name] = fs.watch( - name, {persistent: false}, - cfreader.on_watch_event(name, type, options, cb)); - } - catch (e) { - if (e.code !== 'ENOENT') { // ignore error when ENOENT - console.error(`Error watching config file: ${name} : ${e}`); - } - else { - cfreader._enoent_files[name] = true; - cfreader.ensure_enoent_timer(); - } - } - return; -} + get_cache_key (name, options) { -cfreader.get_cache_key = (name, options) => { + // Ignore options etc. if this is an overriden value + if (this._overrides[name]) return name; - // Ignore options etc. if this is an overriden value - if (cfreader._overrides[name]) return name; + if (options) { + // ordering of objects isn't guaranteed to be consistent, but typically is. + return name + JSON.stringify(options); + } - if (options) { - // this ordering of objects isn't guaranteed to be consistent, but I've - // heard that it typically is. - return name + JSON.stringify(options); - } + if (this._read_args[name] && this._read_args[name].options) { + return name + JSON.stringify(this._read_args[name].options); + } - if (cfreader._read_args[name] && cfreader._read_args[name].options) { - return name + JSON.stringify(cfreader._read_args[name].options); + return name; } - return name; -} + read_config (name, type, cb, options) { + // Store arguments used so we can: + // 1. re-use them by filename later + // 2. to know which files we've read, so we can ignore + // other files written to the same directory. + + this._read_args[name] = { + type, + cb, + options + }; + + // Check cache first + if (!process.env.WITHOUT_CONFIG_CACHE) { + const cache_key = this.get_cache_key(name, options); + // console.log(`\tcache_key: ${cache_key}`); + if (this._config_cache[cache_key] !== undefined) { + // console.log(`\t${name} is cached`); + return this._config_cache[cache_key]; + } + } -cfreader.read_config = (name, type, cb, options) => { - // Store arguments used so we can: - // 1. re-use them by filename later - // 2. to know which files we've read, so we can ignore - // other files written to the same directory. - - cfreader._read_args[name] = { - type, - cb, - options - }; - - // Check cache first - if (!process.env.WITHOUT_CONFIG_CACHE) { - const cache_key = cfreader.get_cache_key(name, options); - // console.log(`\tcache_key: ${cache_key}`); - if (cfreader._config_cache[cache_key] !== undefined) { - // console.log(`\t${name} is cached`); - return cfreader._config_cache[cache_key]; + // load config file + const result = this.load_config(name, type, options); + if (!this.watch_files) return result; + + // We can watch the directory on these platforms which + // allows us to notice when files are newly created. + switch (process.platform) { + case 'win32': + case 'win64': + case 'linux': + this.watch_dir(); + break; + default: + // All other operating systems + this.watch_file(name, type, cb, options); } - } - // load config file - const result = cfreader.load_config(name, type, options); - if (!cfreader.watch_files) return result; - - // We can watch the directory on these platforms which - // allows us to notice when files are newly created. - switch (process.platform) { - case 'win32': - case 'win64': - case 'linux': - cfreader.watch_dir(); - break; - default: - // All other operating systems - cfreader.watch_file(name, type, cb, options); + return result; } - return result; -} - -function isDirectory (filepath) { - return new Promise(function (resolve, reject) { - fs.stat(filepath, (err, stat) => { - if (err) return reject(err); - resolve(stat.isDirectory()); + read_dir (name, opts) { + return new Promise((resolve, reject) => { + + this._read_args[name] = { opts } + const type = opts.type || 'binary'; + + isDirectory(name) + .then((result) => { + return fsReadDir(name); + }) + .then((fileList) => { + const reader = require(path.resolve(__dirname, 'readers', type)); + const promises = []; + for (const file of fileList) { + promises.push(reader.loadPromise(path.resolve(name, file))) + } + return Promise.all(promises); + }) + .then((fileList) => { + // console.log(fileList); + resolve(fileList); + }) + .catch(reject) + + if (opts.watchCb) this.fsWatchDir(name); }) - }) -} + } -function fsReadDir (filepath) { - return new Promise(function (resolve, reject) { - fs.readdir(filepath, (err, fileList) => { - if (err) return reject(err); - resolve(fileList); - }) - }) -} + ensure_enoent_timer () { + if (this._enoent_timer) return; + // Create timer + this._enoent_timer = setInterval(() => { + const files = Object.keys(this._enoent_files); + for (const fileOuter of files) { + /* BLOCK SCOPE */ + ((file) => { + fs.stat(file, (err) => { + if (err) return; + // File now exists + delete(this._enoent_files[file]); + const args = this._read_args[file]; + this.load_config(file, args.type, args.options, args.cb); + this._watchers[file] = fs.watch(file, {persistent: false}, + this.on_watch_event(file, args.type, args.options, args.cb)); + }); + })(fileOuter); // END BLOCK SCOPE + } + }, 60 * 1000); + this._enoent_timer.unref(); // This shouldn't block exit + } -function fsWatchDir (dirPath) { + get_filetype_reader (type) { + switch (type) { + case 'list': + case 'value': + case 'data': + case '': + return require(path.resolve(__dirname, 'readers', 'flat')); + } + return require(path.resolve(__dirname, 'readers', type)); + } - if (cfreader._watchers[dirPath]) return; + load_config (name, type, options) { + let result; - cfreader._watchers[dirPath] = fs.watch(dirPath, { persistent: false, recursive: true }, (fse, filename) => { - // console.log(`event: ${fse}, ${filename}`); - if (!filename) return; - const full_path = path.join(dirPath, filename); - const args = cfreader._read_args[dirPath]; - // console.log(args); - if (cfreader._sedation_timers[full_path]) { - clearTimeout(cfreader._sedation_timers[full_path]); + if (!type) { + type = path.extname(name).toLowerCase().substring(1); } - cfreader._sedation_timers[full_path] = setTimeout(() => { - delete cfreader._sedation_timers[full_path]; - args.opts.watchCb(); - }, 2 * 1000); - }); -} -cfreader.read_dir = (name, opts, done) => { + let cfrType = this.get_filetype_reader(type); - cfreader._read_args[name] = { opts } - const type = opts.type || 'binary'; + if (!fs.existsSync(name)) { + if (!/\.h?json$/.test(name)) { + return cfrType.empty(options, type); + } - isDirectory(name) - .then((result) => { - return fsReadDir(name); - }) - .then((fileList) => { - const reader = require(path.resolve(__dirname, 'readers', type)); - const promises = []; - fileList.forEach((file) => { - promises.push(reader.loadPromise(path.resolve(name, file))) - }); - return Promise.all(promises); - }) - .then((fileList) => { - // console.log(fileList); - done(null, fileList); - }) - .catch((error) => { - done(error); - }) + const yaml_name = name.replace(/\.h?json$/, '.yaml'); + if (!fs.existsSync(yaml_name)) return cfrType.empty(options, type); - if (opts.watchCb) fsWatchDir(name); -} + name = yaml_name; + type = 'yaml'; -cfreader.ensure_enoent_timer = () => { - if (cfreader._enoent_timer) return; - // Create timer - cfreader._enoent_timer = setInterval(() => { - const files = Object.keys(cfreader._enoent_files); - for (let i=0; i { - if (err) return; - // File now exists - delete(cfreader._enoent_files[file]); - const args = cfreader._read_args[file]; - cfreader.load_config(file, args.type, args.options, args.cb); - cfreader._watchers[file] = fs.watch( - file, {persistent: false}, - cfreader.on_watch_event( - file, args.type, args.options, args.cb)); - }); - })(fileOuter); // END BLOCK SCOPE + cfrType = this.get_filetype_reader(type); } - }, 60 * 1000); - cfreader._enoent_timer.unref(); // This shouldn't block exit -} - -cfreader.get_filetype_reader = (type) => { - switch (type) { - case 'list': - case 'value': - case 'data': - case '': - return require(path.resolve(__dirname, 'readers', 'flat')); - } - return require(path.resolve(__dirname, 'readers', type)); -} - -cfreader.load_config = (name, type, options) => { - let result; - if (!type) { - type = path.extname(name).toLowerCase().substring(1); + const cache_key = this.get_cache_key(name, options); + try { + switch (type) { + case 'ini': + result = cfrType.load(name, options, this.regex); + break; + case 'hjson': + case 'json': + case 'yaml': + result = cfrType.load(name); + this.process_file_overrides(name, options, result); + break; + // case 'binary': + default: + result = cfrType.load(name, type, options, this.regex); + } + this._config_cache[cache_key] = result; + } + catch (err) { + console.error(err.message); + if (this._config_cache[cache_key]) { + return this._config_cache[cache_key]; + } + return cfrType.empty(options, type); + } + return result; } - let cfrType = cfreader.get_filetype_reader(type); + process_file_overrides (name, options, result) { + // We might be re-loading this file: + // * build a list of cached overrides + // * remove them and add them back + const cp = this.config_path; + const cache_key = this.get_cache_key(name, options); - if (!fs.existsSync(name)) { - if (!/\.h?json$/.test(name)) { - return cfrType.empty(options, type); + if (this._config_cache[cache_key]) { + for (const ck in this._config_cache[cache_key]) { + if (ck.substr(0,1) === '!') delete this._config_cache[path.join(cp, ck.substr(1))]; + } } - const yaml_name = name.replace(/\.h?json$/, '.yaml'); - if (!fs.existsSync(yaml_name)) return cfrType.empty(options, type); + // Allow JSON files to create or overwrite other config file data + // by prefixing the outer variable name with ! e.g. !smtp.ini + for (const key in result) { + if (key.substr(0,1) !== '!') continue; + const fn = key.substr(1); + // Overwrite the config cache for this filename + console.log(`Overriding file ${fn} with config from ${name}`); + this._config_cache[path.join(cp, fn)] = result[key]; + } + } - name = yaml_name; - type = 'yaml'; + fsWatchDir (dirPath) { - cfrType = cfreader.get_filetype_reader(type); - } + if (this._watchers[dirPath]) return; + const watchOpts = { persistent: false, recursive: true } - const cache_key = cfreader.get_cache_key(name, options); - try { - switch (type) { - case 'ini': - result = cfrType.load(name, options, regex); - break; - case 'hjson': - case 'json': - case 'yaml': - result = cfrType.load(name); - cfreader.process_file_overrides(name, options, result); - break; - // case 'binary': - default: - result = cfrType.load(name, type, options, regex); - } - cfreader._config_cache[cache_key] = result; - } - catch (err) { - console.error(err.message); - if (cfreader._config_cache[cache_key]) { - return cfreader._config_cache[cache_key]; - } - return cfrType.empty(options, type); + this._watchers[dirPath] = fs.watch(dirPath, watchOpts, (fse, filename) => { + // console.log(`event: ${fse}, ${filename}`); + if (!filename) return; + const full_path = path.join(dirPath, filename); + const args = this._read_args[dirPath]; + // console.log(args); + if (this._sedation_timers[full_path]) { + clearTimeout(this._sedation_timers[full_path]); + } + this._sedation_timers[full_path] = setTimeout(() => { + delete this._sedation_timers[full_path]; + args.opts.watchCb(); + }, 2 * 1000); + }); } - return result; } -cfreader.process_file_overrides = (name, options, result) => { - // We might be re-loading this file: - // * build a list of cached overrides - // * remove them and add them back - const cp = cfreader.config_path; - const cache_key = cfreader.get_cache_key(name, options); - if (cfreader._config_cache[cache_key]) { - const ck_keys = Object.keys(cfreader._config_cache[cache_key]); - for (let i=0; i { + fs.stat(filepath, (err, stat) => { + if (err) return reject(err); + resolve(stat.isDirectory()); + }) + }) +} + +function fsReadDir (filepath) { + return new Promise((resolve, reject) => { + fs.readdir(filepath, (err, fileList) => { + if (err) return reject(err); + resolve(fileList); + }) + }) } diff --git a/package.json b/package.json index 6f48ed6..f0127db 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "name": "haraka-config", "license": "MIT", "description": "Haraka's config file loader", - "version": "1.0.18", + "version": "1.0.19", "homepage": "http://haraka.github.io", "repository": { "type": "git", @@ -11,7 +11,7 @@ }, "main": "config.js", "engines": { - "node": ">= 6" + "node": ">= 10" }, "dependencies": { "js-yaml": "^3.13.1" @@ -29,9 +29,9 @@ "url": "https://github.com/haraka/haraka-config/issues" }, "scripts": { - "test": "./node_modules/.bin/mocha test test/readers", - "lint": "./node_modules/.bin/eslint *.js readers/*.js test/*.js test/*/*.js", - "lintfix": "./node_modules/.bin/eslint --fix *.js readers/*.js test/*.js test/*/*.js", - "cover": "./node_modules/.bin/nyc --reporter=lcov ./node_modules/.bin/mocha test test/readers" + "test": "npx mocha test test/readers", + "lint": "npx eslint *.js readers/*.js test/*.js test/*/*.js", + "lintfix": "npx eslint --fix *.js readers/*.js test/*.js test/*/*.js", + "cover": "npx nyc --reporter=lcov npm test" } } diff --git a/test/configfile.js b/test/configfile.js index 54cd007..1b648b2 100644 --- a/test/configfile.js +++ b/test/configfile.js @@ -1,7 +1,6 @@ 'use strict'; const assert = require('assert') - // const path = require('path'); function _setUp (done) {