Skip to content

Rewrite PythonInterpreter. #2106

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
astroshim wants to merge 26 commits into from
Closed

Rewrite PythonInterpreter. #2106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
7304919
initialize python interpreter using py4j
Mar 7, 2017
276011e
add py4j lib
Mar 7, 2017
1395875
removed unnecessary code.
Mar 7, 2017
c3f5b78
Merge branch 'master' of https://github.com/apache/zeppelin into py4j…
Mar 8, 2017
af097ac
add testcase
Mar 8, 2017
f17bff4
fix testcase failure.
Mar 8, 2017
5ae5120
fix interpreter-setting
Mar 8, 2017
cbbc15c
fix py4j path
Mar 8, 2017
a50179e
add conda interpreter
Mar 8, 2017
3c9585f
update interpreter-setting.json
Mar 8, 2017
a48df58
Merge branch 'master' into py4jPythonInterpreter
Mar 8, 2017
574bd21
fix interpreter-setting error
Mar 8, 2017
60e9820
bug fix about copying library
Mar 8, 2017
e49ad24
add pandas
Mar 13, 2017
046db88
add testcase
Mar 14, 2017
f8e19be
fix matplotlib testcase
Mar 14, 2017
be5db4d
fix pandas sql testcase
Mar 15, 2017
e8570d2
fix ci for pandassql
Mar 15, 2017
ac92cdb
fix python interpreter testcase
Mar 15, 2017
aad7ee8
fix testcase
Mar 15, 2017
8a016c9
Merge branch 'master' into py4jPythonInterpreter
Mar 15, 2017
9fcf144
Make python docker interpreter work using py4j
Leemoonsoo Mar 17, 2017
2eb5de7
Fix PythonDockerInterpreterTest.java test
Leemoonsoo Mar 17, 2017
a76b0d8
fix test on python3
Leemoonsoo Mar 17, 2017
e511ebe
add PythonDockerInterpreter to interpreter-setting.json
Leemoonsoo Mar 17, 2017
c9b195b
Merge pull request #16 from Leemoonsoo/py4jdocker
Mar 18, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions python/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,43 @@
</executions>
</plugin>

<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>wagon-maven-plugin</artifactId>
<version>1.0</version>
<executions>
<execution>
<phase>package</phase>
<goals><goal>download-single</goal></goals>
<configuration>
<url>https://pypi.python.org/packages/64/5c/01e13b68e8caafece40d549f232c9b5677ad1016071a48d04cc3895acaa3</url>
<fromFile>py4j-${py4j.version}.zip</fromFile>
<toFile>${project.build.directory}/../../interpreter/python/py4j-${py4j.version}.zip</toFile>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
<executions>
<execution>
<phase>package</phase>
<configuration>
<target>
<unzip src="${project.build.directory}/../../interpreter/python/py4j-${py4j.version}.zip"
dest="${project.build.directory}/../../interpreter/python"/>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>


<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
Expand Down
37 changes: 31 additions & 6 deletions python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.*;
import java.nio.file.Paths;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand All @@ -37,14 +35,19 @@ public class PythonDockerInterpreter extends Interpreter {
Pattern activatePattern = Pattern.compile("activate\\s*(.*)");
Pattern deactivatePattern = Pattern.compile("deactivate");
Pattern helpPattern = Pattern.compile("help");
private File zeppelinHome;

public PythonDockerInterpreter(Properties property) {
super(property);
}

@Override
public void open() {

if (System.getenv("ZEPPELIN_HOME") != null) {
zeppelinHome = new File(System.getenv("ZEPPELIN_HOME"));
} else {
zeppelinHome = Paths.get("..").toAbsolutePath().toFile();
}
}

@Override
Expand All @@ -54,6 +57,7 @@ public void close() {

@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
File pythonScript = new File(getPythonInterpreter().getScriptPath());
InterpreterOutput out = context.out;

Matcher activateMatcher = activatePattern.matcher(st);
Expand All @@ -66,7 +70,27 @@ public InterpreterResult interpret(String st, InterpreterContext context) {
} else if (activateMatcher.matches()) {
String image = activateMatcher.group(1);
pull(out, image);
setPythonCommand("docker run -i --rm " + image + " python -iu");

// mount pythonscript dir
String mountPythonScript = "-v " +
pythonScript.getParentFile().getAbsolutePath() +
":/_zeppelin_tmp ";

// mount zeppelin dir
String mountPy4j = "-v " +
zeppelinHome.getAbsolutePath() +
":/_zeppelin ";

// set PYTHONPATH
String pythonPath = ":/_zeppelin/" + PythonInterpreter.ZEPPELIN_PY4JPATH + ":" +
":/_zeppelin/" + PythonInterpreter.ZEPPELIN_PYTHON_LIBS;

setPythonCommand("docker run -i --rm " +
mountPythonScript +
mountPy4j +
"-e PYTHONPATH=\"" + pythonPath + "\" " +
image +
" python /_zeppelin_tmp/" + pythonScript.getName());
restartPythonProcess();
out.clear();
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "\"" + image + "\" activated");
Expand All @@ -79,6 +103,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) {
}
}


public void setPythonCommand(String cmd) {
PythonInterpreter python = getPythonInterpreter();
python.setPythonCommand(cmd);
Expand Down
Loading