Skip to content

ZEPPELIN-497 ] pyspark completion #530

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cloverhearts wants to merge 5 commits into from
Closed

ZEPPELIN-497 ] pyspark completion #530

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ab05f86
implement pyspark completion
cloverhearts Dec 11, 2015
f4dc487
Pyspark completion __ filtering
cloverhearts Dec 16, 2015
1f82825
remove variable resultComplaetion
cloverhearts Dec 16, 2015
d7efe88
change filter regex character
cloverhearts Dec 17, 2015
6832409
Changed PySpark Completion Filter regex
cloverhearts Dec 17, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 91 additions & 2 deletions spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;

import py4j.GatewayServer;

/**
Expand Down Expand Up @@ -368,12 +373,96 @@ public int getProgress(InterpreterContext context) {
return sparkInterpreter.getProgress(context);
}


@Override
public List<String> completion(String buf, int cursor) {
// not supported
return new LinkedList<String>();
if (buf.length() < cursor) {
cursor = buf.length();
}
String completionString = getCompletionTargetString(buf, cursor);
String completionCommand = "completion.getCompletion('" + completionString + "')";

//start code for completion
SparkInterpreter sparkInterpreter = getSparkInterpreter();
if (sparkInterpreter.getSparkVersion().isUnsupportedVersion() == false
&& pythonscriptRunning == false) {
return new LinkedList<String>();
}

outputStream.reset();

pythonInterpretRequest = new PythonInterpretRequest(completionCommand, "");
statementOutput = null;

synchronized (statementSetNotifier) {
statementSetNotifier.notify();
}

synchronized (statementFinishedNotifier) {
while (statementOutput == null) {
try {
statementFinishedNotifier.wait(1000);
} catch (InterruptedException e) {
// not working
logger.info("wait drop");
return new LinkedList<String>();
}
}
}

if (statementError) {
return new LinkedList<String>();
}
InterpreterResult completionResult = new InterpreterResult(Code.SUCCESS, statementOutput);
//end code for completion

Gson gson = new Gson();

return gson.fromJson(completionResult.message(), LinkedList.class);
}

private String getCompletionTargetString(String text, int cursor) {
String[] completionSeqCharaters = {" ", "\n", "\t"};
int completionEndPosition = cursor;
int completionStartPosition = cursor;
int indexOfReverseSeqPostion = cursor;

String resultCompletionText = "";
String completionScriptText = "";
try {
completionScriptText = text.substring(0, cursor);
}
catch (Exception e) {
logger.error(e.toString());
return null;
}
completionEndPosition = completionScriptText.length();

String tempReverseCompletionText = new StringBuilder(completionScriptText).reverse().toString();

for (String seqCharacter : completionSeqCharaters) {
indexOfReverseSeqPostion = tempReverseCompletionText.indexOf(seqCharacter);

if (indexOfReverseSeqPostion < completionStartPosition && indexOfReverseSeqPostion > 0) {
completionStartPosition = indexOfReverseSeqPostion;
}

}

if (completionStartPosition == completionEndPosition) {
completionStartPosition = 0;
}
else
{
completionStartPosition = completionEndPosition - completionStartPosition;
}
resultCompletionText = completionScriptText.substring(
completionStartPosition , completionEndPosition);

return resultCompletionText;
}


private SparkInterpreter getSparkInterpreter() {
InterpreterGroup intpGroup = getInterpreterGroup();
LazyOpenInterpreter lazy = null;
Expand Down
47 changes: 46 additions & 1 deletion spark/src/main/resources/python/zeppelin_pyspark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
#

import sys, getopt, traceback
import sys, getopt, traceback, json, re

from py4j.java_gateway import java_import, JavaGateway, GatewayClient
from py4j.protocol import Py4JJavaError
Expand Down Expand Up @@ -107,6 +107,50 @@ def isAutoConvertEnabled(self):
def isImportAllPackageUnderSparkSql(self):
return self.version >= self.SPARK_1_3_0

class PySparkCompletion:
def getGlobalCompletion(self):
objectDefList = []
try:
for completionItem in list(globals().iterkeys()):
objectDefList.append(completionItem)
except:
return None
else:
return objectDefList

def getMethodCompletion(self, text_value):
objectDefList = []
completion_target = text_value
try:
if len(completion_target) <= 0:
return None
if text_value[-1] == ".":
Copy link
Member

@felixcheung felixcheung Dec 13, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(actually it looks like dir() should not be given a string, so please ignore my earlier comment here.)

Copy link
Member

@felixcheung felixcheung Dec 13, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also you might want to filter out private methods prefixed __ or _

>>> dir(a)
['__class__', '__cmp__', '__contains__', '__delattr__', '__delitem__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__gt__', '__hash__', '__init__', '__iter__', '__le__', '__len__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setitem__', '__sizeof__', '__str__', '__subclasshook__', 'clear', 'copy', 'fromkeys', 'get', 'has_key', 'items', 'iteritems', 'iterkeys', 'itervalues', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values', 'viewitems', 'viewkeys', 'viewvalues']

Copy link
Member Author

@cloverhearts cloverhearts Dec 13, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@felixcheung
Thank you for Feed back.
You need to filter the __ private method?
I think filtering is unnecessary for __.
This is because in the private method __ to think you can use a class that you have created.
If you ever need a filter, you can hear more reasons?

  • my English is not very good. Thank you good feedback -

Copy link
Member

@felixcheung felixcheung Dec 13, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it is possible that you want to call a private method named starting with __ - but generally that is discouraged in python

In the following cases, autocomplete returns a list including __add__ and other private method, so maybe it shouldn't?

image

image

Copy link
Member

@felixcheung felixcheung Dec 13, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also they are not callable like a.__add__ - their names are mangled even though dir() returns them: http://www.diveintopython.net/object_oriented_framework/private_functions.html
"You can access the __parse method of the MP3FileInfo class by the name _MP3FileInfo__parse. Acknowledge that this is interesting, but promise to never, ever do it in real code. Private methods are private for a reason, but like many other things in Python, their privateness is ultimately a matter of convention, not force."

Copy link
Member Author

@cloverhearts cloverhearts Dec 14, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I understand.
I think the good feedback.
And by that omission in the auto-complete because it should not call,
I hope to add a filtering operation for __.
Thank you for feedback.

completion_target = text_value[:-1]
exec("%s = %s(%s)" % ("objectDefList", "dir", completion_target))
except:
return None
else:
return objectDefList


def getCompletion(self, text_value):
completionList = set()

globalCompletionList = self.getGlobalCompletion()
if globalCompletionList != None:
for completionItem in list(globalCompletionList):
completionList.add(completionItem)

if text_value != None:
objectCompletionList = self.getMethodCompletion(text_value)
Copy link
Member

@felixcheung felixcheung Dec 13, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if text_value is None should it returned all objects/names in the current scope (dir())?

if objectCompletionList != None:
for completionItem in list(objectCompletionList):
completionList.add(completionItem)
if len(completionList) <= 0:
print ""
else:
print json.dumps(filter(lambda x : not re.match("^__.*", x), list(completionList)))


output = Logger()
sys.stdout = output
Expand Down Expand Up @@ -149,6 +193,7 @@ def isImportAllPackageUnderSparkSql(self):
sqlc = SQLContext(sc, intp.getSQLContext())
sqlContext = sqlc

completion = PySparkCompletion()
z = PyZeppelinContext(intp.getZeppelinContext())

while True :
Expand Down
29 changes: 1 addition & 28 deletions zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,35 +540,8 @@ angular.module('zeppelinWebApp')

pos = session.getTextRange(new Range(0, 0, pos.row, pos.column)).length;
var buf = session.getValue();
var completionString = buf;

if (pos > 0) {
var completionStartPosition = pos;
var completionSeqCharaters = [' ', '\n'];

// replace \r\n or \n\r other to \n
var reverseCompletionString = buf.replace(/\r?\n|\r/g, '\n').substr(0, pos).split('').reverse();
for (var seqCharacterIndex in completionSeqCharaters) {
var indexOfReverseSeqPostion = reverseCompletionString.indexOf(completionSeqCharaters[seqCharacterIndex]);

if (indexOfReverseSeqPostion < completionStartPosition && indexOfReverseSeqPostion > 0) {
completionStartPosition = indexOfReverseSeqPostion;
}
}

if (completionStartPosition === pos) {
completionStartPosition = 0;
}
else
{
completionStartPosition = pos - completionStartPosition;
}

completionString = buf.substr( completionStartPosition , pos);
pos = completionString.length -1;
}

websocketMsgSrv.completion($scope.paragraph.id, completionString, pos);
websocketMsgSrv.completion($scope.paragraph.id, buf, pos);

$scope.$on('completionList', function(event, data) {
if (data.completions) {
Expand Down