Truncates long streaming job confs (#1773)

When running a streaming job with lots of inputs, the job conf gets passed to
each mapper. This can cause an exception from passing too many arguments to the
mapper. This job conf is not actually, needed, so it's safe to truncate. 20000
is recommended as a safe value for this truncation in
http://aajisaka.github.io/hadoop-project/hadoop-streaming/HadoopStreaming.html

luigi/contrib/hadoop.py

@@ -764,6 +764,7 @@ def on_failure(self, exception):
 
 
 class JobTask(BaseHadoopJobTask):
+    jobconf_truncate = 20000
     n_reduce_tasks = 25
     reducer = NotImplemented
 
@@ -773,6 +774,8 @@ def jobconfs(self):
             jcs.append('mapred.reduce.tasks=0')
         else:
             jcs.append('mapred.reduce.tasks=%s' % self.n_reduce_tasks)
+        if self.jobconf_truncate >= 0:
+            jcs.append('stream.jobconf.truncate.limit=%i' % self.jobconf_truncate)
         return jcs
 
     def init_mapper(self):