diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index edfa5e3d31c..2a1819e8ef3 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -92,5 +92,11 @@ commons-lang3 3.3.2 + + org.apache.commons + commons-jexl + 2.1.1 + + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Evaluator.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Evaluator.java new file mode 100644 index 00000000000..0ab40db6f0e --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Evaluator.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.display; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.jexl2.Expression; +import org.apache.commons.jexl2.JexlContext; +import org.apache.commons.jexl2.JexlEngine; +import org.apache.commons.jexl2.MapContext; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The evaluator helper class. + * + * This class is initialized using the fully qualified name of an + * utility class having one or more static methods. + * + * Example:com.company.custom.udf.UDFUtility + * + * Utility class is resolved using ZEPPELIN_UTILITY_CLASS nv variable or + * 'zeppelin.utility.class' JVM property. + * + * When an expression of the type: "eval:doSomething(...)" is passed from Zeppelin, + * the Evaluator class tries to resolve the something(...) method in the utility class provided + * at initialization time. + * + * Passing an empty utility class at initialization time also works, but then it is mandatory + * to use the fully qualified name when writing the expression in Zeppelin. + * + * Example: eval:com.company.custom.udf.UDFUtility.doSomething(...) + * + * The command coming from Zeppelin notebook is evaluated using 'commons-jexl'. + * + */ +public class Evaluator { + + public static final String EVAL_PREFIX = "eval:"; + private static final int EVAL_PREFIX_LENGTH = EVAL_PREFIX.length(); + private static Pattern REGEX = Pattern.compile("(?.+\\..+)\\.(?.+)"); + + + private static Logger LOG = LoggerFactory.getLogger(Evaluator.class); + Class utilityClass; + + /** + * Constructs an evaluator class given an user-defined Utility class fully qualified name. + * + * @param classImpl Utility class containing. + * @throws ClassNotFoundException if utility class is not in the classpath. + */ + public Evaluator(String classImpl) throws ClassNotFoundException { + if (StringUtils.isEmpty(classImpl)) + LOG.debug("Only full qualified expressions will be executed... Be careful!"); + else + this.utilityClass = Class.forName(classImpl); + } + + /** + * Evaluates the given command coming verbatim from Zeppelin notebook. + * + * The command coming from Zeppelin notebook is evaluated using 'commons-jexl'. + * + * @param command expression to eval + * @return the result of + */ + public Object eval(String command) throws UnsupportedOperationException { + + Object obj = null; + + // Check if expression has to be evaluated. + if (!command.startsWith(EVAL_PREFIX)) { + return command; + } + + String expressionToEval = command.substring(EVAL_PREFIX_LENGTH); + + try { + Map mapFQN = getFQN(expressionToEval); + + // First, we try with fully qualified name + JexlEngine jexl = new JexlEngine(); + String expression = "utils." + mapFQN.get("method"); + Expression expr = jexl.createExpression(expression); + JexlContext jc = new MapContext(); + + jc.set("utils", Class.forName(mapFQN.get("clazz"))); + obj = expr.evaluate(jc); + if (obj != null) + return obj; + } catch (Exception e) { + LOG.debug("Error trying to use the FQN class."); + } + + // If it fails, we apply the default utility class + LOG.debug("Trying the default utility class"); + try { + JexlEngine jexl = new JexlEngine(); + String expression = "utils." + expressionToEval; + Expression expr = jexl.createExpression(expression); + JexlContext jc = new MapContext(); + jc.set("utils", utilityClass); + + obj = expr.evaluate(jc); + } catch (Exception e) { + LOG.debug("Error using configured utility class"); + throw new UnsupportedOperationException("Could not evaluate expression."); + } + + if (obj == null) + throw new UnsupportedOperationException("Could not evaluate expression."); + + return obj; + } + + /** + * This method extracts the class and the method with arguments from a fully qualified class. + * + * @param function FQN class to interpret + * @return Map with both clazz and method to execute + */ + private static HashMap getFQN(String function) { + HashMap map = new HashMap<>(); + Matcher matcher = REGEX.matcher(function); + + matcher.find(); + + String matcherClazz = matcher.group("clazz"); + String matcherMethod = matcher.group("method"); + + map.put("clazz", matcherClazz); + map.put("method", matcherMethod); + + return map; + } + +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java index 2f7858ca03c..3630c8516e1 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java @@ -20,9 +20,11 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -87,7 +89,7 @@ public Input(String name, Object defaultValue, ParamOption[] options) { this.options = options; } - + public Input(String name, String displayName, String type, Object defaultValue, ParamOption[] options, boolean hidden) { super(); @@ -300,6 +302,65 @@ public static String getSimpleQuery(Map params, String script) { } + /** + * This method is similar to getSimpleQuery. Main differences are that params + * argument is not changed at all, and the method look for expressions to eval + * on the params map (the value will starts with 'eval:') + * @param params + * @param script + * @param classUtility This is the utility class used to eval expressions. The utility + * class should define the statisc methods to interpret + * @return + * @throws Exception + */ + public static String getSimpleQueryForEvaluation(Map params, String script + , String classUtility) throws Exception { + String replaced = script; + Evaluator evaluator = new Evaluator(classUtility); + + // Recorremos todos los parametros evaluando sii es posible + Iterator entrySet = params.entrySet().iterator(); + Map aux = new HashMap(); + while (entrySet.hasNext()) { + Entry thisEntry = (Entry) entrySet.next(); + String key = thisEntry.getKey().toString(); + String value = thisEntry.getValue().toString(); + aux.put(key, evaluator.eval(value)); + } + + for (String key : aux.keySet()) { + Object value = aux.get(key); + if (value == null) continue; + replaced = replaced.replaceAll("[_]?[$][{]([^:]*[:])?" + key + + "([(][^)]*[)])?(=[^}]*)?[}]", value.toString()); + } + + Pattern pattern = Pattern.compile("[$][{]([^=}]*[=][^}]*)[}]"); + while (true) { + Matcher match = pattern.matcher(replaced); + if (match != null && match.find()) { + String m = match.group(1); + int p = m.indexOf('='); + String replacement = m.substring(p + 1); + int optionP = replacement.indexOf(","); + if (optionP > 0) { + replacement = replacement.substring(0, optionP); + } + replaced = replaced.replaceFirst( + "[_]?[$][{]" + + m.replaceAll("[(]", ".").replaceAll("[)]", ".") + .replaceAll("[|]", ".") + "[}]", replacement); + } else { + break; + } + } + + replaced = replaced.replace("[_]?[$][{]([^=}]*)[}]", ""); + return replaced; + } + + + public static String[] split(String str) { return str.split(";(?=([^\"']*\"[^\"']*\")*[^\"']*$)"); @@ -443,7 +504,7 @@ && getBlockStr(blockStart[b]).compareTo(str.substring(blockStartPos, i)) == 0) { // check if block is started for (int b = 0; b < blockStart.length; b++) { if (curString.substring(lastEscapeOffset + 1) - .endsWith(getBlockStr(blockStart[b])) == true) { + .endsWith(getBlockStr(blockStart[b]))) { blockStack.add(0, b); // block is started blockStartPos = i; break; diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/RegexForFQNTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/RegexForFQNTest.java new file mode 100644 index 00000000000..544e69be177 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/RegexForFQNTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.display; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import junit.framework.Assert; + +import org.junit.Test; + +public class RegexForFQNTest { + + @Test + public void test() { + String function = "org.keedio.sample.Utility.suma(1,2)"; + String clazz = "org.keedio.sample.Utility"; + String method = "suma(1,2)"; + + String pattern = "(?.+\\..+)\\.(?.+)"; + + Pattern regex = Pattern.compile(pattern); + Matcher matcher = regex.matcher(function); + + matcher.find(); + + String matcherClazz = matcher.group("clazz"); + String matcherMethod = matcher.group("method"); + + Assert.assertTrue(clazz.equals(matcherClazz)); + Assert.assertTrue(method.equals(matcherMethod)); + } + +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 33a427c25e1..bbdcba08bf5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -17,11 +17,13 @@ package org.apache.zeppelin.notebook; +import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.display.Input; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.Interpreter.FormType; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.JobListener; import org.slf4j.Logger; @@ -29,6 +31,9 @@ import java.io.Serializable; import java.util.*; +import java.util.Map.Entry; + +import static org.apache.zeppelin.display.Evaluator.EVAL_PREFIX; /** * Paragraph is a representation of an execution unit. @@ -37,9 +42,12 @@ */ public class Paragraph extends Job implements Serializable, Cloneable { private static final transient long serialVersionUID = -6328572073497992016L; + private transient NoteInterpreterLoader replLoader; private transient Note note; + private static Logger LOG = LoggerFactory.getLogger(Paragraph.class); + String title; String text; Date dateUpdated; @@ -201,13 +209,55 @@ protected Object jobRun() throws Throwable { Map inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built // from script body settings.setForms(inputs); - script = Input.getSimpleQuery(settings.getParams(), scriptBody); + + if (needsEvaluation(settings.getParams())) { + /* + * User introduced the string "eval:doSomething(...)" in the notebook. We need to + * evaluate the expression first. + */ + + try { + // Resolve the user-provided utility class + String utilityClass = ZeppelinConfiguration.create() + .getString("ZEPPELIN_UTILITY_CLASS", "zeppelin.utility.class", ""); + + script = Input.getSimpleQueryForEvaluation(settings.getParams(), + scriptBody, utilityClass); + } catch (UnsupportedOperationException e) { + LOG.debug("Error while evaluating: " + settings.getParams()); + InterpreterResult ret = new InterpreterResult(Code.ERROR, e.getMessage()); + + return ret; + } + } else { + // no evaluation needed, default behaviour. + script = Input.getSimpleQuery(settings.getParams(), scriptBody); + } + } logger().info("RUN : " + script); InterpreterResult ret = repl.interpret(script, getInterpreterContext()); return ret; } + /** + * Checks if at least one of the given arguments needs to be evaluated. + * + * @param map the map of arguments passed by the user in the zeppelin notebook. + * @return true if at least one params starts with "eval:", false otherwise. + */ + private boolean needsEvaluation(Map map) { + boolean ret = false; + + for (Object o : map.entrySet()) { + String value = (String) ((Entry) o).getValue(); + ret = ret || value.startsWith(EVAL_PREFIX); + } + + return ret; + + } + @Override protected boolean jobAbort() { Interpreter repl = getRepl(getRequiredReplName());