From 3d0a9d4c3781dce264b186aa862d8dddeb52518d Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Sun, 22 May 2016 14:42:48 +0900 Subject: [PATCH 1/6] Implement feat. dynamic load interpreter class --- conf/zeppelin-env.cmd.template | 1 + conf/zeppelin-env.sh.template | 1 + .../zeppelin/rest/InterpreterRestApi.java | 39 +++++++++ .../LoadDynamicInterpreterRequest.java | 56 ++++++++++++ .../zeppelin/conf/ZeppelinConfiguration.java | 1 + .../interpreter/InterpreterFactory.java | 85 ++++++++++++++++++- 6 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/LoadDynamicInterpreterRequest.java diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template index 06799b5fd85..7afa2b65c78 100644 --- a/conf/zeppelin-env.cmd.template +++ b/conf/zeppelin-env.cmd.template @@ -34,6 +34,7 @@ REM set ZEPPELIN_NOTEBOOK_S3_USER REM User in bucket where notebook REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default. REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0. REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading +REM set ZEPPELIN_INTERPRETER_DOWNLOAD_DIR REM download interpreter for directory path REM Spark interpreter configuration diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 6279de72139..1c41478b943 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -35,6 +35,7 @@ # export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default. # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading +# export ZEPPELIN_INTERPRETER_DOWNLOAD_DIR # download interpreter for directory path #### Spark interpreter configuration #### diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index 64309730e51..4f23d0c419b 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -36,6 +36,7 @@ import org.apache.zeppelin.dep.Repository; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; +import org.apache.zeppelin.rest.message.LoadDynamicInterpreterRequest; import org.apache.zeppelin.rest.message.NewInterpreterSettingRequest; import org.apache.zeppelin.rest.message.UpdateInterpreterSettingRequest; import org.apache.zeppelin.server.JsonResponse; @@ -239,4 +240,42 @@ public Response removeRepository(@PathParam("repoId") String repoId) { } return new JsonResponse(Status.OK).build(); } + + /** + * load a downloaded interpreter via external repository + */ + @POST + @Path("load/{interpreterGroupName}/{interpreterName}") + public Response loadDynamicInterpreter( + @PathParam("interpreterGroupName") String interpreterGroupName, + @PathParam("interpreterName") String interpreterName, String message) { + logger.info("dynamic load interpreter interpreterGroupName [{}] name [{}]", + interpreterGroupName, interpreterName); + try { + LoadDynamicInterpreterRequest request = gson.fromJson( + message, LoadDynamicInterpreterRequest.class + ); + + if (request.getClassName() == null + || request.getArtifact() == null + || request.getUrl() == null) { + throw new Exception("invalid request data"); + } + + interpreterFactory.loadDynamicInterpreter( + interpreterGroupName, + interpreterName, + request.getArtifact(), + request.getClassName(), + request.getUrl(), + request.isSnapshot() + ); + + } catch (Exception e) { + logger.error("Exception in InterpreterRestApi while adding repository - load failed ", e); + return new JsonResponse( + Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); + } + return new JsonResponse(Status.OK).build(); + } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/LoadDynamicInterpreterRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/LoadDynamicInterpreterRequest.java new file mode 100644 index 00000000000..658768be5b6 --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/LoadDynamicInterpreterRequest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.rest.message; + +import java.util.Map; + +/** + * LoadDynamicInterpreterRequest rest api request message + */ + +public class LoadDynamicInterpreterRequest { + String artifact; + String className; + Map repository; + + public LoadDynamicInterpreterRequest() { + + } + + public String getArtifact() { return artifact; } + + public String getClassName() { return className; } + + public Map getRepository() { return repository; } + + public String getUrl() throws ClassCastException { + Object urlObj = repository.get("url"); + if (urlObj == null) { + return null; + } + return (String) urlObj; + } + + public Boolean isSnapshot() throws ClassCastException { + Object snapshotFlagObj = repository.get("snapshot"); + if (snapshotFlagObj == null) { + return false; + } + return (Boolean) snapshotFlagObj; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 541aae1397b..5d3815dd8c0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -503,6 +503,7 @@ public static enum ConfVars { // Decide when new note is created, interpreter settings will be binded automatically or not. ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true), ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), + ZEPPELIN_INTERPRETER_DOWNLOAD_DIR("zeppelin.interpreter.download.dir", "interpreter"), ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"), // Allows a way to specify a ',' separated list of allowed origins for rest and websockets // i.e. http://localhost:8080 diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 08e64656d65..b1bc214ff6b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -60,7 +60,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { .synchronizedMap(new HashMap()); private ZeppelinConfiguration conf; - String[] interpreterClassList; + List interpreterClassList; private Map interpreterSettings = new HashMap(); @@ -99,7 +99,10 @@ public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultO this.interpreterRepositories = depResolver.getRepos(); this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS); - interpreterClassList = replsConf.split(","); + interpreterClassList = new ArrayList(); + for (String className : replsConf.split(",")) { + interpreterClassList.add(className); + } GsonBuilder builder = new GsonBuilder(); builder.setPrettyPrinting(); @@ -110,6 +113,84 @@ public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultO init(); } + public boolean loadDynamicInterpreter(String intpGroupName, String intpName, String artifact, + String intpClassName) { + return loadDynamicInterpreter(intpGroupName, intpName, artifact, intpClassName, null, false); + } + + public boolean loadDynamicInterpreter(String intpGroupName, String intpName, String artifact, + String intpClassName, String repositoryUrl, boolean isSnapShotRepo) { + String[] artifactItem = artifact.split(":"); + String zepInterpreterRepoDir = conf.getString(ConfVars.ZEPPELIN_INTERPRETER_DOWNLOAD_DIR); + String zepInterpreterRepoFullPath = conf.getRelativeDir( + ConfVars.ZEPPELIN_INTERPRETER_DOWNLOAD_DIR + ); + String interpreterDesPath = String.format("%s/%s/%s/", zepInterpreterRepoDir, + intpGroupName, intpName); + String interpreterLoadPath = String.format("%s/%s/%s", zepInterpreterRepoFullPath, + intpGroupName, intpName); + + if (artifactItem.length <= 0) { + logger.error("Failed load dynamic interpreter - invalid artifact : {}", artifact); + return false; + } + + try { + if (repositoryUrl != null) { + depResolver.addRepo("dyInterpreterRepo", repositoryUrl, isSnapShotRepo); + } + logger.info("interpreter path : {}", interpreterLoadPath); + depResolver.load(artifact, interpreterDesPath); + setDynamicInterpreter(intpClassName, interpreterLoadPath); + } catch (Exception e) { + logger.error("Failed load dynamic interpreter : ", e); + return false; + } + return true; + } + + protected void setDynamicInterpreter(String interpreterClassName, String fileDirPath) + throws InterpreterException, IOException { + logger.info("load Dynamic Interpreter ClassName : {}", interpreterClassName); + logger.info("load Dynamic Interpreter FilePath : {}", interpreterClassName); + + ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); + interpreterClassList.add(interpreterClassName); + // Load classes + File interpreterDir = new File(fileDirPath); + + if (interpreterDir != null) { + URL[] urls = null; + try { + urls = recursiveBuildLibList(interpreterDir); + } catch (MalformedURLException e1) { + logger.error("Can't load jars ", e1); + } + URLClassLoader ccl = new URLClassLoader(urls, oldcl); + + try { + Class.forName(interpreterClassName, true, ccl); + Set keys = Interpreter.registeredInterpreters.keySet(); + for (String intName : keys) { + if (interpreterClassName.equals( + Interpreter.registeredInterpreters.get(intName).getClassName())) { + Interpreter.registeredInterpreters.get(intName).setPath(fileDirPath); + logger.info("Interpreter {} found. class={}", intName, fileDirPath); + cleanCl.put(fileDirPath, ccl); + } + } + } catch (ClassNotFoundException e) { + logger.error("Load error : ", e); + } + } + + for (String settingId : interpreterSettings.keySet()) { + InterpreterSetting setting = interpreterSettings.get(settingId); + logger.info("Interpreter setting group {} : id={}, name={}", + setting.getGroup(), settingId, setting.getName()); + } + } + private void init() throws InterpreterException, IOException, RepositoryException { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); From 599639fb0ced60e26807e9f771fe8f127945596c Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Mon, 23 May 2016 14:17:49 +0900 Subject: [PATCH 2/6] fixed invalid result for dynamic interpreter --- .../java/org/apache/zeppelin/rest/InterpreterRestApi.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index 4f23d0c419b..53c5460f007 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -262,7 +262,7 @@ public Response loadDynamicInterpreter( throw new Exception("invalid request data"); } - interpreterFactory.loadDynamicInterpreter( + boolean result = interpreterFactory.loadDynamicInterpreter( interpreterGroupName, interpreterName, request.getArtifact(), @@ -271,10 +271,14 @@ public Response loadDynamicInterpreter( request.isSnapshot() ); + if (result == false) { + throw new Exception("can't not found artifact"); + } + } catch (Exception e) { logger.error("Exception in InterpreterRestApi while adding repository - load failed ", e); return new JsonResponse( - Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); + Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); } return new JsonResponse(Status.OK).build(); } From 3965f740787a7acd5a6219c5228fa3fdc2109c76 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Sat, 4 Jun 2016 22:49:52 +0900 Subject: [PATCH 3/6] add config in zeppelin env - dynamic interperter --- conf/zeppelin-env.cmd.template | 2 +- conf/zeppelin-env.sh.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template index f80ca19708c..d4be68cf45d 100644 --- a/conf/zeppelin-env.cmd.template +++ b/conf/zeppelin-env.cmd.template @@ -34,7 +34,7 @@ REM set ZEPPELIN_NOTEBOOK_S3_USER REM User in bucket where notebook REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default. REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0. REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading -REM set ZEPPELIN_INTERPRETER_DOWNLOAD_DIR REM download interpreter for directory path +REM set ZEPPELIN_INTERPRETER_DOWNLOAD_DIR REM Path for interpreter download REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index d917fe4467b..7fc616d28f7 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -35,7 +35,7 @@ # export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default. # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading -# export ZEPPELIN_INTERPRETER_DOWNLOAD_DIR # download interpreter for directory path +# export ZEPPELIN_INTERPRETER_DOWNLOAD_DIR # Path for interpreter download # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). #### Spark interpreter configuration #### From 1bb3a0c138e38e602ed487fa59075c557687df62 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Sat, 4 Jun 2016 23:05:46 +0900 Subject: [PATCH 4/6] add path for interpreter download path at doc/install/install.md --- docs/install/install.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/install/install.md b/docs/install/install.md index 696f837b295..e110e0f978a 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -174,6 +174,12 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin notebook The root directory where Zeppelin notebook directories are saved + + ZEPPELIN_INTERPRETER_DOWNLOAD_DIR + zeppelin.interpreter.download.dir + interpreter + Path for interpreter download + ZEPPELIN_NOTEBOOK_S3_BUCKET zeppelin.notebook.s3.bucket From 377393aad7a36c919b8f86875f7aac30708e14ab Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Sat, 4 Jun 2016 23:23:25 +0900 Subject: [PATCH 5/6] add interpreter download dir for zeppelin-site.xml.template --- conf/zeppelin-site.xml.template | 6 ++++++ .../org/apache/zeppelin/conf/ZeppelinConfiguration.java | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 2d1fea1ba36..101bd618a22 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -170,6 +170,12 @@ Interpreter implementation base directory + + zeppelin.interpreter.download.dir + interpreter + Path for interpreter download + + zeppelin.interpreter.localRepo local-repo diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 4633bbe795c..75c517478ab 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -508,6 +508,7 @@ public static enum ConfVars { ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"), + ZEPPELIN_INTERPRETER_DOWNLOAD_DIR("zeppelin.interpreter.download.dir", "interpreter"), ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000), ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10), ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"), @@ -531,7 +532,6 @@ public static enum ConfVars { // Decide when new note is created, interpreter settings will be binded automatically or not. ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true), ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), - ZEPPELIN_INTERPRETER_DOWNLOAD_DIR("zeppelin.interpreter.download.dir", "interpreter"), ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"), // Allows a way to specify a ',' separated list of allowed origins for rest and websockets // i.e. http://localhost:8080 From 04d84d30ff57cb2c8d1c63e95233e2ffa3eae4f8 Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Thu, 9 Jun 2016 14:53:55 +0900 Subject: [PATCH 6/6] add exception for dynamic loading --- .../zeppelin/interpreter/InterpreterFactory.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index e5c8af50555..d410cf7fa1d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -22,8 +22,10 @@ import com.google.gson.reflect.TypeToken; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOExceptionWithCause; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.NullArgumentException; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.dep.Dependency; @@ -156,7 +158,7 @@ public boolean loadDynamicInterpreter(String intpGroupName, String intpName, Str } protected void setDynamicInterpreter(String interpreterClassName, String fileDirPath) - throws InterpreterException, IOException { + throws InterpreterException, IOException, ClassNotFoundException { logger.info("load Dynamic Interpreter ClassName : {}", interpreterClassName); logger.info("load Dynamic Interpreter FilePath : {}", interpreterClassName); @@ -169,8 +171,9 @@ protected void setDynamicInterpreter(String interpreterClassName, String fileDir URL[] urls = null; try { urls = recursiveBuildLibList(interpreterDir); - } catch (MalformedURLException e1) { - logger.error("Can't load jars ", e1); + } catch (MalformedURLException e) { + logger.error("Can't load jars ", e); + throw new IOException(e); } URLClassLoader ccl = new URLClassLoader(urls, oldcl); @@ -187,6 +190,7 @@ protected void setDynamicInterpreter(String interpreterClassName, String fileDir } } catch (ClassNotFoundException e) { logger.error("Load error : ", e); + throw e; } }