diff --git a/.gitignore b/.gitignore index b785247..b84710b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules npm-debug.log tmp +test/output/ diff --git a/lib/parallelizer.js b/lib/parallelizer.js index a19410a..16d0308 100644 --- a/lib/parallelizer.js +++ b/lib/parallelizer.js @@ -4,6 +4,9 @@ var path = require('path'); var util = require('util'); var async = require('async'); var _ = require('lodash'); +var uuid = require('uuid'); +var fs = require('fs'); +var os = require('os'); var lpad = require('./lpad'); @@ -29,32 +32,41 @@ Parallelizer.prototype.exec = function(task, target) { // spawnOptions = { stdio: 'inherit' }; // } - lpad(' '); - var ok = true; - async.forEach(splittedFilesSrc, function(filesSrc, next) { - var filesOption = '--grunt-parallelize-child-filesSrc=' + filesSrc.join(path.delimiter); - var cp = this.grunt_.util.spawn({ - grunt: true, - args: [['parallelize', task, target].join(':')].concat(this.grunt_.option.flags(), filesOption), - opts: spawnOptions - }, function(err, result, code) { - if ((err || code > 0)) { - ok = false; - if (result.stderr) { - this.grunt_.warn(result.stderr); - } + var self = this; + // create /tmp if not exists + fs.mkdir(os.tmpdir(), function (err) { + lpad(' '); + var ok = true; + async.forEach(splittedFilesSrc, function(filesSrc, next) { + // need to put filesSrc in a temp file because of command line arg data limit + var tmpFile = path.join(os.tmpdir(), task + target + uuid.v4()); + fs.writeFile(tmpFile, JSON.stringify(filesSrc), function (err) { + if (err) { throw err; } + var filesOption = '--grunt-parallelize-child-filesSrc=' + tmpFile; + var cp = self.grunt_.util.spawn({ + grunt: true, + args: [['parallelize', task, target].join(':')].concat(self.grunt_.option.flags(), filesOption), + opts: spawnOptions + }, function(err, result, code) { + if ((err || code > 0)) { + ok = false; + if (result.stderr) { + self.grunt_.warn(result.stderr); + } + } + self.grunt_.log.write('\n' + self.filterOutput_(result.stdout, task, target) + '\n'); + next(); + }); + self.cpCache_.push(cp); + }); + }, function() { + lpad(); + if (!ok) { + self.grunt_.log.writeln(''); } - this.grunt_.log.write('\n' + this.filterOutput_(result.stdout, task, target) + '\n'); - next(); - }.bind(this)); - this.cpCache_.push(cp); - }.bind(this), function() { - lpad(); - if (!ok) { - this.grunt_.log.writeln(''); - } - cb(ok); - }.bind(this)); + cb(ok); + }); + }); }; Parallelizer.prototype.filterOutput_ = function(output, task, target) { @@ -101,7 +113,23 @@ Parallelizer.prototype.getNormalizedFilesSrc_ = function(task, target) { var configPath = [task, target]; var config = this.grunt_.config(configPath); var files = this.grunt_.task.normalizeMultiTaskFiles(config); - return _(files).chain().pluck('src').flatten().uniq().value(); + // if any file objects have a destination, can assume that not all src files + // can be merged into single list + var hasDest = false; + files.forEach(function (fileObj) { + if (fileObj.dest !== undefined){ hasDest = true; } + }); + var normalizedFiles; + if (hasDest){ + normalizedFiles = files; + } else { + normalizedFiles = _(files).chain().pluck('src').flatten().uniq().value(); + normalizedFiles = normalizedFiles.map(function (srcFile) { + return { src: srcFile, }; + }); + } + + return normalizedFiles; }; module.exports = Parallelizer; diff --git a/lib/taskrunner.js b/lib/taskrunner.js index cafb938..5f20270 100644 --- a/lib/taskrunner.js +++ b/lib/taskrunner.js @@ -1,31 +1,27 @@ 'use strict'; -var _ = require('lodash'); -var path = require('path'); +var fs = require('fs'); module.exports = function(grunt, childFilesSrcOption, task, target) { - var childFilesSrc = childFilesSrcOption.split(path.delimiter); - var configPath = [task, target]; - var config = grunt.config(configPath); - var kindOf = grunt.util.kindOf; - var type = kindOf(config); - if (type === 'array') { - grunt.config(configPath, childFilesSrc); - } else if (type === 'object') { - if (_.contains(['string', 'array'], kindOf(config.src))) { - // Compact Format - grunt.config(configPath.concat('src'), childFilesSrc); - } else if ('array' === kindOf(config.files)) { - // Files Array Format - grunt.config(configPath.concat('files'), [{src: childFilesSrc}]); - } else if ('object' === kindOf(config.files)) { - // Files Object Format - throw new Error('The config of ' + task + ' is Files Object Format.'); - } else { - throw new Error('The config of ' + task + ' is not supported.'); + var done = grunt.task.current.async(); + fs.readFile(childFilesSrcOption, function (err, data) { + if (err) { throw err; } + var childFilesSrc = JSON.parse(data); + var configPath = [task, target]; + + // replace the original file config with the smaller file set assigned by + // the parallelizer + var targetCfg = grunt.config.get(configPath); + if (targetCfg.hasOwnProperty('dest')){ + delete targetCfg.dest; + } + if (targetCfg.hasOwnProperty('src')){ + delete targetCfg.src; } - } else { - throw new Error('The config of ' + task + ' is not supported.'); - } - grunt.task.run(configPath.join(':')); + targetCfg.files = childFilesSrc; + grunt.config.set(configPath, targetCfg); + + grunt.task.run(configPath.join(':')); + done(); + }); }; diff --git a/package.json b/package.json index 7f2c6da..8295451 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "dependencies": { "async": "~0.9.0", "lodash": "~3.1.0", - "lpad": "~1.0.0" + "lpad": "~1.0.0", + "uuid": "^2.0.1" }, "devDependencies": { "expect.js": "~0.3.1", diff --git a/test/cases/compactFormatWithDest.Gruntfile.js b/test/cases/compactFormatWithDest.Gruntfile.js new file mode 100644 index 0000000..a80ddf2 --- /dev/null +++ b/test/cases/compactFormatWithDest.Gruntfile.js @@ -0,0 +1,25 @@ +'use strict'; + +module.exports = function(grunt) { + grunt.initConfig({ + writefilesrc: { + test_src: { + dest: '../output/compactFormatWithDest-1.txt', + src: '../fixtures/*.txt', + } + }, + + parallelize: { + writefilesrc: { + test_src: 2, + } + } + }); + + // Load this tasks. + grunt.loadTasks('../../tasks'); + // Load tasks for testing. + grunt.loadTasks('../tasks'); + // Set defaut task. + grunt.registerTask('default', ['parallelize:writefilesrc:test_src']); +}; diff --git a/test/cases/filesArrayFormatWithDest.Gruntfile.js b/test/cases/filesArrayFormatWithDest.Gruntfile.js new file mode 100644 index 0000000..f79abde --- /dev/null +++ b/test/cases/filesArrayFormatWithDest.Gruntfile.js @@ -0,0 +1,33 @@ +'use strict'; + +module.exports = function(grunt) { + grunt.initConfig({ + writefilesrc: { + test: { + files: [ + { dest: '../output/filesArrayFormatWithDest-1.txt', src: '../fixtures/*.txt', }, + ], + }, + test_multi: { + files: [ + { dest: '../output/filesArrayFormatWithDest-2.txt', src: '../fixtures/1.txt', }, + { dest: '../output/filesArrayFormatWithDest-3.txt', src: ['../fixtures/3.txt', '../fixtures/4.txt'], }, + ], + }, + }, + + parallelize: { + writefilesrc: { + test: 2, + test_multi: 2, + } + } + }); + + // Load this tasks. + grunt.loadTasks('../../tasks'); + // Load tasks for testing. + grunt.loadTasks('../tasks'); + // Set defaut task. + grunt.registerTask('default', ['parallelize:writefilesrc:test', 'parallelize:writefilesrc:test_multi']); +}; diff --git a/test/cases/filesObjectFormatWithDest.Gruntfile.js b/test/cases/filesObjectFormatWithDest.Gruntfile.js new file mode 100644 index 0000000..2d12d9e --- /dev/null +++ b/test/cases/filesObjectFormatWithDest.Gruntfile.js @@ -0,0 +1,33 @@ +'use strict'; + +module.exports = function(grunt) { + grunt.initConfig({ + writefilesrc: { + test: { + files: { + '../output/filesObjectFormatWithDest-1.txt': '../fixtures/*.txt', + }, + }, + test_multi: { + files: { + '../output/filesObjectFormatWithDest-2.txt': '../fixtures/1.txt', + '../output/filesObjectFormatWithDest-3.txt': ['../fixtures/3.txt', '../fixtures/4.txt'], + }, + }, + }, + + parallelize: { + writefilesrc: { + test: 2, + test_multi: 2, + } + } + }); + + // Load this tasks. + grunt.loadTasks('../../tasks'); + // Load tasks for testing. + grunt.loadTasks('../tasks'); + // Set defaut task. + grunt.registerTask('default', ['parallelize:writefilesrc:test', 'parallelize:writefilesrc:test_multi']); +}; diff --git a/test/fixtures/file_output/compactFormatWithDest-1.txt b/test/fixtures/file_output/compactFormatWithDest-1.txt new file mode 100644 index 0000000..aa1b300 --- /dev/null +++ b/test/fixtures/file_output/compactFormatWithDest-1.txt @@ -0,0 +1 @@ +["../fixtures/1.txt","../fixtures/2.txt","../fixtures/3.txt","../fixtures/4.txt"] diff --git a/test/fixtures/file_output/filesArrayFormatWithDest-1.txt b/test/fixtures/file_output/filesArrayFormatWithDest-1.txt new file mode 100644 index 0000000..aa1b300 --- /dev/null +++ b/test/fixtures/file_output/filesArrayFormatWithDest-1.txt @@ -0,0 +1 @@ +["../fixtures/1.txt","../fixtures/2.txt","../fixtures/3.txt","../fixtures/4.txt"] diff --git a/test/fixtures/file_output/filesArrayFormatWithDest-2.txt b/test/fixtures/file_output/filesArrayFormatWithDest-2.txt new file mode 100644 index 0000000..f74e623 --- /dev/null +++ b/test/fixtures/file_output/filesArrayFormatWithDest-2.txt @@ -0,0 +1 @@ +["../fixtures/1.txt"] diff --git a/test/fixtures/file_output/filesArrayFormatWithDest-3.txt b/test/fixtures/file_output/filesArrayFormatWithDest-3.txt new file mode 100644 index 0000000..0790c7a --- /dev/null +++ b/test/fixtures/file_output/filesArrayFormatWithDest-3.txt @@ -0,0 +1 @@ +["../fixtures/3.txt","../fixtures/4.txt"] diff --git a/test/fixtures/file_output/filesObjectFormatWithDest-1.txt b/test/fixtures/file_output/filesObjectFormatWithDest-1.txt new file mode 100644 index 0000000..aa1b300 --- /dev/null +++ b/test/fixtures/file_output/filesObjectFormatWithDest-1.txt @@ -0,0 +1 @@ +["../fixtures/1.txt","../fixtures/2.txt","../fixtures/3.txt","../fixtures/4.txt"] diff --git a/test/fixtures/file_output/filesObjectFormatWithDest-2.txt b/test/fixtures/file_output/filesObjectFormatWithDest-2.txt new file mode 100644 index 0000000..f74e623 --- /dev/null +++ b/test/fixtures/file_output/filesObjectFormatWithDest-2.txt @@ -0,0 +1 @@ +["../fixtures/1.txt"] diff --git a/test/fixtures/file_output/filesObjectFormatWithDest-3.txt b/test/fixtures/file_output/filesObjectFormatWithDest-3.txt new file mode 100644 index 0000000..0790c7a --- /dev/null +++ b/test/fixtures/file_output/filesObjectFormatWithDest-3.txt @@ -0,0 +1 @@ +["../fixtures/3.txt","../fixtures/4.txt"] diff --git a/test/parallelize_test.js b/test/parallelize_test.js index 60c30f5..528a353 100644 --- a/test/parallelize_test.js +++ b/test/parallelize_test.js @@ -32,7 +32,7 @@ describe('grunt-parallelize', function() { }); describe('grunt parallelize', function() { - this.timeout(5000); + this.timeout(10000); it('runs all tasks', function(done) { testGruntfile('runAllTasks', done); }); @@ -48,8 +48,44 @@ describe('grunt-parallelize', function() { }); }); + describe('Writes files', function() { + it('Compact Format', function(done){ + testGruntfileWithFileWrite('compactFormatWithDest', done); + }); + + it('Files Array Format', function(done){ + testGruntfileWithFileWrite('filesArrayFormatWithDest', done); + }); + + it('Files Object Format', function(done){ + testGruntfileWithFileWrite('filesObjectFormatWithDest', done); + }); + }); }); +function testGruntfileWithFileWrite(name, callback){ + var prefix = __dirname + '/cases/' + name; + var gruntfile = prefix + '.Gruntfile.js'; + var expectedDir = __dirname + '/fixtures/file_output/'; + var expectedFiles = []; + fs.readdirSync(expectedDir).forEach(function(file){ + if (file.indexOf(name) === 0) { + expectedFiles.push(file); + } + }); + var outputDir = __dirname + '/output/'; + // clean up the output dir + deleteFolderRecursive(outputDir); + + runGruntfile(gruntfile, function(err, stdout, stderr) { + expectedFiles.forEach(function(file){ + expect(fs.readFileSync(outputDir + file, {encoding: 'utf8'})) + .to.be(fs.readFileSync(expectedDir + file, {encoding: 'utf8'})); + }); + callback(err); + }); +} + function testGruntfile(name, callback) { var prefix = __dirname + '/cases/' + name; var gruntfile = prefix + '.Gruntfile.js'; @@ -65,7 +101,8 @@ function testGruntfile(name, callback) { return; } try { - expect(stdout).to.be(expected); + expect(stdout.split('\n').sort().join('\n')) + .to.be(expected.split('\n').sort().join('\n')); } catch (e) { console.log('expected:'); console.log(expected); @@ -82,3 +119,19 @@ function runGruntfile(gruntfile, callback) { var options = {}; exec(cmd, options, callback); } + +function deleteFolderRecursive(path) { + var files = []; + if( fs.existsSync(path) ) { + files = fs.readdirSync(path); + files.forEach(function(file,index){ + var curPath = path + '/' + file; + if(fs.lstatSync(curPath).isDirectory()) { // recurse + deleteFolderRecursive(curPath); + } else { // delete file + fs.unlinkSync(curPath); + } + }); + fs.rmdirSync(path); + } +} diff --git a/test/tasks/writefilesrc.js b/test/tasks/writefilesrc.js new file mode 100644 index 0000000..ecad1dd --- /dev/null +++ b/test/tasks/writefilesrc.js @@ -0,0 +1,34 @@ +/* + * grunt-parallelize + * https://github.com/teppeis/grunt-parallelize + * + * Copyright (c) 2013 Teppei Sato + * Licensed under the MIT license. + */ +'use strict'; + +/* + * Write src files to dest file for test. + */ +function createMultiTask(grunt, name) { + grunt.registerMultiTask(name, 'Check the files exits.', function() { + var done = this.async(); + grunt.util.async.forEachSeries(this.files, function(file, next) { + if (!file.dest) { + throw new Error('Missing dest'); + } + + grunt.file.write(file.dest, JSON.stringify(file.src) + '\n', {encoding: 'utf8'}); + next(); + }, function(err) { + done(err); + }); + }); +} + +module.exports = function(grunt) { + createMultiTask(grunt, 'writefilesrc'); +}; + +module.exports.createMultiTask = createMultiTask; +