diff --git a/luigi/contrib/hadoop.py b/luigi/contrib/hadoop.py index 80d1a74c08..7004dd409c 100644 --- a/luigi/contrib/hadoop.py +++ b/luigi/contrib/hadoop.py @@ -764,6 +764,7 @@ def on_failure(self, exception): class JobTask(BaseHadoopJobTask): + jobconf_truncate = 20000 n_reduce_tasks = 25 reducer = NotImplemented @@ -773,6 +774,8 @@ def jobconfs(self): jcs.append('mapred.reduce.tasks=0') else: jcs.append('mapred.reduce.tasks=%s' % self.n_reduce_tasks) + if self.jobconf_truncate >= 0: + jcs.append('stream.jobconf.truncate.limit=%i' % self.jobconf_truncate) return jcs def init_mapper(self):