From 29df40dab9963e0dbce4119bdd872a86ff670af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=BE=E6=9E=97=E8=A5=BF?= Date: Tue, 28 Jun 2016 20:37:05 +0800 Subject: [PATCH] [SPARK-16253][SQL] make spark sql compatible with hive sql that using python script transform like using 'xxx.py' --- .../spark/sql/hive/execution/ScriptTransformation.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala index d9b9ba4bfdfe..7c5f468dc872 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala @@ -67,6 +67,14 @@ case class ScriptTransformation( val cmd = List("/bin/bash", "-c", script) val builder = new ProcessBuilder(cmd.asJava) + // for compatibility with hive + // if the script is a python/bash/... file, we should make it executable + if (new File(script).exists) { + new File(script).setExecutable(true) + val env = builder.environment() + env.put("PATH", env.get("PATH") + ":.") + } + val proc = builder.start() val inputStream = proc.getInputStream val outputStream = proc.getOutputStream