Support All file formats and pass file config in temp file

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 npm-debug.log
 tmp
+test/output/

lib/parallelizer.js

@@ -4,6 +4,9 @@ var path = require('path');
 var util = require('util');
 var async = require('async');
 var _ = require('lodash');
+var uuid = require('uuid');
+var fs = require('fs');
+var os = require('os');
 
 var lpad = require('./lpad');
 
@@ -29,32 +32,41 @@ Parallelizer.prototype.exec = function(task, target) {
   //   spawnOptions = { stdio: 'inherit' };
   // }
 
-  lpad('    ');
-  var ok = true;
-  async.forEach(splittedFilesSrc, function(filesSrc, next) {
-    var filesOption = '--grunt-parallelize-child-filesSrc=' + filesSrc.join(path.delimiter);
-    var cp = this.grunt_.util.spawn({
-      grunt: true,
-      args: [['parallelize', task, target].join(':')].concat(this.grunt_.option.flags(), filesOption),
-      opts: spawnOptions
-    }, function(err, result, code) {
-      if ((err || code > 0)) {
-        ok = false;
-        if (result.stderr) {
-          this.grunt_.warn(result.stderr);
-        }
+  var self = this;
+  // create /tmp if not exists
+  fs.mkdir(os.tmpdir(), function (err) {
+    lpad('    ');
+    var ok = true;
+    async.forEach(splittedFilesSrc, function(filesSrc, next) {
+      // need to put filesSrc in a temp file because of command line arg data limit
+      var tmpFile = path.join(os.tmpdir(), task + target + uuid.v4());
+      fs.writeFile(tmpFile, JSON.stringify(filesSrc), function (err) {
+        if (err) { throw err; }
+        var filesOption = '--grunt-parallelize-child-filesSrc=' + tmpFile;
+        var cp = self.grunt_.util.spawn({
+          grunt: true,
+          args: [['parallelize', task, target].join(':')].concat(self.grunt_.option.flags(), filesOption),
+          opts: spawnOptions
+        }, function(err, result, code) {
+          if ((err || code > 0)) {
+            ok = false;
+            if (result.stderr) {
+              self.grunt_.warn(result.stderr);
+            }
+          }
+          self.grunt_.log.write('\n' + self.filterOutput_(result.stdout, task, target) + '\n');
+          next();
+        });
+        self.cpCache_.push(cp);
+      });
+    }, function() {
+      lpad();
+      if (!ok) {
+        self.grunt_.log.writeln('');
       }
-      this.grunt_.log.write('\n' + this.filterOutput_(result.stdout, task, target) + '\n');
-      next();
-    }.bind(this));
-    this.cpCache_.push(cp);
-  }.bind(this), function() {
-    lpad();
-    if (!ok) {
-      this.grunt_.log.writeln('');
-    }
-    cb(ok);
-  }.bind(this));
+      cb(ok);
+    });
+  });
 };
 
 Parallelizer.prototype.filterOutput_ = function(output, task, target) {
@@ -101,7 +113,23 @@ Parallelizer.prototype.getNormalizedFilesSrc_ = function(task, target) {
   var configPath = [task, target];
   var config = this.grunt_.config(configPath);
   var files = this.grunt_.task.normalizeMultiTaskFiles(config);
-  return _(files).chain().pluck('src').flatten().uniq().value();
+  // if any file objects have a destination, can assume that not all src files
+  // can be merged into single list
+  var hasDest = false;
+  files.forEach(function (fileObj) {
+    if (fileObj.dest !== undefined){ hasDest = true; }
+  });
+  var normalizedFiles;
+  if (hasDest){
+    normalizedFiles = files;
+  } else {
+    normalizedFiles = _(files).chain().pluck('src').flatten().uniq().value();
+    normalizedFiles = normalizedFiles.map(function (srcFile) {
+      return { src: srcFile, };
+    });
+  }
+
+  return normalizedFiles;
 };
 
 module.exports = Parallelizer;

lib/taskrunner.js

@@ -1,31 +1,27 @@
 'use strict';
 
-var _ = require('lodash');
-var path = require('path');
+var fs = require('fs');
 
 module.exports = function(grunt, childFilesSrcOption, task, target) {
-  var childFilesSrc = childFilesSrcOption.split(path.delimiter);
-  var configPath = [task, target];
-  var config = grunt.config(configPath);
-  var kindOf = grunt.util.kindOf;
-  var type = kindOf(config);
-  if (type === 'array') {
-    grunt.config(configPath, childFilesSrc);
-  } else if (type === 'object') {
-    if (_.contains(['string', 'array'], kindOf(config.src))) {
-      // Compact Format
-      grunt.config(configPath.concat('src'), childFilesSrc);
-    } else if ('array' === kindOf(config.files)) {
-      // Files Array Format
-      grunt.config(configPath.concat('files'), [{src: childFilesSrc}]);
-    } else if ('object' === kindOf(config.files)) {
-      // Files Object Format
-      throw new Error('The config of ' + task + ' is Files Object Format.');
-    } else {
-      throw new Error('The config of ' + task + ' is not supported.');
+  var done = grunt.task.current.async();
+  fs.readFile(childFilesSrcOption, function (err, data) {
+    if (err) { throw err; }
+    var childFilesSrc = JSON.parse(data);
+    var configPath = [task, target];
+
+    // replace the original file config with the smaller file set assigned by
+    // the parallelizer
+    var targetCfg = grunt.config.get(configPath);
+    if (targetCfg.hasOwnProperty('dest')){
+      delete targetCfg.dest;
+    }
+    if (targetCfg.hasOwnProperty('src')){
+      delete targetCfg.src;
     }
-  } else {
-    throw new Error('The config of ' + task + ' is not supported.');
-  }
-  grunt.task.run(configPath.join(':'));
+    targetCfg.files = childFilesSrc;
+    grunt.config.set(configPath, targetCfg);
+
+    grunt.task.run(configPath.join(':'));
+    done();
+  });
 };

package.json

@@ -34,7 +34,8 @@
   "dependencies": {
     "async": "~0.9.0",
     "lodash": "~3.1.0",
-    "lpad": "~1.0.0"
+    "lpad": "~1.0.0",
+    "uuid": "^2.0.1"
   },
   "devDependencies": {
     "expect.js": "~0.3.1",

test/cases/compactFormatWithDest.Gruntfile.js

@@ -0,0 +1,25 @@
+'use strict';
+
+module.exports = function(grunt) {
+  grunt.initConfig({
+    writefilesrc: {
+      test_src: {
+        dest: '../output/compactFormatWithDest-1.txt',
+        src: '../fixtures/*.txt',
+      }
+    },
+
+    parallelize: {
+      writefilesrc: {
+        test_src: 2,
+      }
+    }
+  });
+
+  // Load this tasks.
+  grunt.loadTasks('../../tasks');
+  // Load tasks for testing.
+  grunt.loadTasks('../tasks');
+  // Set defaut task.
+  grunt.registerTask('default', ['parallelize:writefilesrc:test_src']);
+};

test/cases/filesArrayFormatWithDest.Gruntfile.js

@@ -0,0 +1,33 @@
+'use strict';
+
+module.exports = function(grunt) {
+  grunt.initConfig({
+    writefilesrc: {
+      test: {
+        files: [
+          { dest: '../output/filesArrayFormatWithDest-1.txt', src: '../fixtures/*.txt', },
+        ],
+      },
+      test_multi: {
+        files: [
+          { dest: '../output/filesArrayFormatWithDest-2.txt', src: '../fixtures/1.txt', },
+          { dest: '../output/filesArrayFormatWithDest-3.txt', src: ['../fixtures/3.txt', '../fixtures/4.txt'], },
+        ],
+      },
+    },
+
+    parallelize: {
+      writefilesrc: {
+        test: 2,
+        test_multi: 2,
+      }
+    }
+  });
+
+  // Load this tasks.
+  grunt.loadTasks('../../tasks');
+  // Load tasks for testing.
+  grunt.loadTasks('../tasks');
+  // Set defaut task.
+  grunt.registerTask('default', ['parallelize:writefilesrc:test', 'parallelize:writefilesrc:test_multi']);
+};

test/cases/filesObjectFormatWithDest.Gruntfile.js

@@ -0,0 +1,33 @@
+'use strict';
+
+module.exports = function(grunt) {
+  grunt.initConfig({
+    writefilesrc: {
+      test: {
+        files: {
+          '../output/filesObjectFormatWithDest-1.txt': '../fixtures/*.txt',
+        },
+      },
+      test_multi: {
+        files: {
+          '../output/filesObjectFormatWithDest-2.txt': '../fixtures/1.txt',
+          '../output/filesObjectFormatWithDest-3.txt': ['../fixtures/3.txt', '../fixtures/4.txt'],
+        },
+      },
+    },
+
+    parallelize: {
+      writefilesrc: {
+        test: 2,
+        test_multi: 2,
+      }
+    }
+  });
+
+  // Load this tasks.
+  grunt.loadTasks('../../tasks');
+  // Load tasks for testing.
+  grunt.loadTasks('../tasks');
+  // Set defaut task.
+  grunt.registerTask('default', ['parallelize:writefilesrc:test', 'parallelize:writefilesrc:test_multi']);
+};

test/fixtures/file_output/compactFormatWithDest-1.txt

@@ -0,0 +1 @@
+["../fixtures/1.txt","../fixtures/2.txt","../fixtures/3.txt","../fixtures/4.txt"]

test/fixtures/file_output/filesArrayFormatWithDest-1.txt

@@ -0,0 +1 @@
+["../fixtures/1.txt","../fixtures/2.txt","../fixtures/3.txt","../fixtures/4.txt"]

test/fixtures/file_output/filesArrayFormatWithDest-2.txt

@@ -0,0 +1 @@
+["../fixtures/1.txt"]

test/fixtures/file_output/filesArrayFormatWithDest-3.txt

@@ -0,0 +1 @@
+["../fixtures/3.txt","../fixtures/4.txt"]

test/fixtures/file_output/filesObjectFormatWithDest-1.txt

@@ -0,0 +1 @@
+["../fixtures/1.txt","../fixtures/2.txt","../fixtures/3.txt","../fixtures/4.txt"]

test/fixtures/file_output/filesObjectFormatWithDest-2.txt

@@ -0,0 +1 @@
+["../fixtures/1.txt"]

test/fixtures/file_output/filesObjectFormatWithDest-3.txt

@@ -0,0 +1 @@
+["../fixtures/3.txt","../fixtures/4.txt"]

test/parallelize_test.js

@@ -32,7 +32,7 @@ describe('grunt-parallelize', function() {
   });
 
   describe('grunt parallelize', function() {
-    this.timeout(5000);
+    this.timeout(10000);
     it('runs all tasks', function(done) {
       testGruntfile('runAllTasks', done);
     });
@@ -48,8 +48,44 @@ describe('grunt-parallelize', function() {
     });
   });
 
+  describe('Writes files', function() {
+    it('Compact Format', function(done){
+      testGruntfileWithFileWrite('compactFormatWithDest', done);
+    });
+
+    it('Files Array Format', function(done){
+      testGruntfileWithFileWrite('filesArrayFormatWithDest', done);
+    });
+
+    it('Files Object Format', function(done){
+      testGruntfileWithFileWrite('filesObjectFormatWithDest', done);
+    });
+  });
 });
 
+function testGruntfileWithFileWrite(name, callback){
+  var prefix = __dirname + '/cases/' + name;
+  var gruntfile = prefix + '.Gruntfile.js';
+  var expectedDir = __dirname + '/fixtures/file_output/';
+  var expectedFiles = [];
+  fs.readdirSync(expectedDir).forEach(function(file){
+    if (file.indexOf(name) === 0) {
+      expectedFiles.push(file);
+    }
+  });
+  var outputDir = __dirname + '/output/';
+  // clean up the output dir
+  deleteFolderRecursive(outputDir);
+
+  runGruntfile(gruntfile, function(err, stdout, stderr) {
+    expectedFiles.forEach(function(file){
+      expect(fs.readFileSync(outputDir + file, {encoding: 'utf8'}))
+        .to.be(fs.readFileSync(expectedDir + file, {encoding: 'utf8'}));
+    });
+    callback(err);
+  });
+}
+
 function testGruntfile(name, callback) {
   var prefix = __dirname + '/cases/' + name;
   var gruntfile = prefix + '.Gruntfile.js';
@@ -65,7 +101,8 @@ function testGruntfile(name, callback) {
       return;
     }
     try {
-      expect(stdout).to.be(expected);
+      expect(stdout.split('\n').sort().join('\n'))
+        .to.be(expected.split('\n').sort().join('\n'));
     } catch (e) {
       console.log('expected:');
       console.log(expected);
@@ -82,3 +119,19 @@ function runGruntfile(gruntfile, callback) {
   var options = {};
   exec(cmd, options, callback);
 }
+
+function deleteFolderRecursive(path) {
+  var files = [];
+  if( fs.existsSync(path) ) {
+    files = fs.readdirSync(path);
+    files.forEach(function(file,index){
+      var curPath = path + '/' + file;
+      if(fs.lstatSync(curPath).isDirectory()) { // recurse
+        deleteFolderRecursive(curPath);
+      } else { // delete file
+        fs.unlinkSync(curPath);
+      }
+    });
+    fs.rmdirSync(path);
+  }
+}

test/tasks/writefilesrc.js

@@ -0,0 +1,34 @@
+/*
+ * grunt-parallelize
+ * https://github.com/teppeis/grunt-parallelize
+ *
+ * Copyright (c) 2013 Teppei Sato <teppeis@gmail.com>
+ * Licensed under the MIT license.
+ */
+'use strict';
+
+/*
+ * Write src files to dest file for test.
+ */
+function createMultiTask(grunt, name) {
+  grunt.registerMultiTask(name, 'Check the files exits.', function() {
+    var done = this.async();
+    grunt.util.async.forEachSeries(this.files, function(file, next) {
+      if (!file.dest) {
+        throw new Error('Missing dest');
+      }
+
+      grunt.file.write(file.dest, JSON.stringify(file.src) + '\n', {encoding: 'utf8'});
+      next();
+    }, function(err) {
+      done(err);
+    });
+  });
+}
+
+module.exports = function(grunt) {
+  createMultiTask(grunt, 'writefilesrc');
+};
+
+module.exports.createMultiTask = createMultiTask;
+