diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index 57eb8514a31..02b993148a9 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -37,6 +37,7 @@ import com.google.gson.Gson; import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.rest.message.RestartInterpreterRequest; import org.apache.zeppelin.utils.SecurityUtils; import org.slf4j.Logger; @@ -61,7 +62,7 @@ public class InterpreterRestApi { private static final Logger logger = LoggerFactory.getLogger(InterpreterRestApi.class); - private InterpreterFactory interpreterFactory; + private InterpreterSettingManager interpreterSettingManager; private NotebookServer notebookServer; Gson gson = new Gson(); @@ -69,9 +70,9 @@ public class InterpreterRestApi { public InterpreterRestApi() { } - public InterpreterRestApi(InterpreterFactory interpreterFactory, - NotebookServer notebookWsServer) { - this.interpreterFactory = interpreterFactory; + public InterpreterRestApi(InterpreterSettingManager interpreterSettingManager, + NotebookServer notebookWsServer) { + this.interpreterSettingManager = interpreterSettingManager; this.notebookServer = notebookWsServer; } @@ -82,7 +83,7 @@ public InterpreterRestApi(InterpreterFactory interpreterFactory, @Path("setting") @ZeppelinApi public Response listSettings() { - return new JsonResponse<>(Status.OK, "", interpreterFactory.get()).build(); + return new JsonResponse<>(Status.OK, "", interpreterSettingManager.get()).build(); } /** @@ -93,7 +94,7 @@ public Response listSettings() { @ZeppelinApi public Response getSetting(@PathParam("settingId") String settingId) { try { - InterpreterSetting setting = interpreterFactory.get(settingId); + InterpreterSetting setting = interpreterSettingManager.get(settingId); if (setting == null) { return new JsonResponse<>(Status.NOT_FOUND).build(); } else { @@ -123,7 +124,7 @@ public Response newSettings(String message) { } Properties p = new Properties(); p.putAll(request.getProperties()); - InterpreterSetting interpreterSetting = interpreterFactory + InterpreterSetting interpreterSetting = interpreterSettingManager .createNewSetting(request.getName(), request.getGroup(), request.getDependencies(), request.getOption(), p); logger.info("new setting created with {}", interpreterSetting.getId()); @@ -144,7 +145,7 @@ public Response updateSetting(String message, @PathParam("settingId") String set try { UpdateInterpreterSettingRequest request = gson.fromJson(message, UpdateInterpreterSettingRequest.class); - interpreterFactory + interpreterSettingManager .setPropertyAndRestart(settingId, request.getOption(), request.getProperties(), request.getDependencies()); } catch (InterpreterException e) { @@ -156,7 +157,7 @@ public Response updateSetting(String message, @PathParam("settingId") String set return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); } - InterpreterSetting setting = interpreterFactory.get(settingId); + InterpreterSetting setting = interpreterSettingManager.get(settingId); if (setting == null) { return new JsonResponse<>(Status.NOT_FOUND, "", settingId).build(); } @@ -171,7 +172,7 @@ public Response updateSetting(String message, @PathParam("settingId") String set @ZeppelinApi public Response removeSetting(@PathParam("settingId") String settingId) throws IOException { logger.info("Remove interpreterSetting {}", settingId); - interpreterFactory.remove(settingId); + interpreterSettingManager.remove(settingId); return new JsonResponse(Status.OK).build(); } @@ -184,12 +185,12 @@ public Response removeSetting(@PathParam("settingId") String settingId) throws I public Response restartSetting(String message, @PathParam("settingId") String settingId) { logger.info("Restart interpreterSetting {}, msg={}", settingId, message); - InterpreterSetting setting = interpreterFactory.get(settingId); + InterpreterSetting setting = interpreterSettingManager.get(settingId); try { RestartInterpreterRequest request = gson.fromJson(message, RestartInterpreterRequest.class); String noteId = request == null ? null : request.getNoteId(); - interpreterFactory.restart(settingId, noteId, SecurityUtils.getPrincipal()); + interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal()); notebookServer.clearParagraphRuntimeInfo(setting); } catch (InterpreterException e) { @@ -209,7 +210,7 @@ public Response restartSetting(String message, @PathParam("settingId") String se @GET @ZeppelinApi public Response listInterpreter(String message) { - Map m = interpreterFactory.getAvailableInterpreterSettings(); + Map m = interpreterSettingManager.getAvailableInterpreterSettings(); return new JsonResponse<>(Status.OK, "", m).build(); } @@ -220,7 +221,7 @@ public Response listInterpreter(String message) { @Path("repository") @ZeppelinApi public Response listRepositories() { - List interpreterRepositories = interpreterFactory.getRepositories(); + List interpreterRepositories = interpreterSettingManager.getRepositories(); return new JsonResponse<>(Status.OK, "", interpreterRepositories).build(); } @@ -235,8 +236,8 @@ public Response listRepositories() { public Response addRepository(String message) { try { Repository request = gson.fromJson(message, Repository.class); - interpreterFactory.addRepository(request.getId(), request.getUrl(), request.isSnapshot(), - request.getAuthentication(), request.getProxy()); + interpreterSettingManager.addRepository(request.getId(), request.getUrl(), + request.isSnapshot(), request.getAuthentication(), request.getProxy()); logger.info("New repository {} added", request.getId()); } catch (Exception e) { logger.error("Exception in InterpreterRestApi while adding repository ", e); @@ -258,7 +259,7 @@ public Response getMetaInfo(@Context HttpServletRequest req, return new JsonResponse<>(Status.BAD_REQUEST).build(); } String propValue = null; - InterpreterSetting interpreterSetting = interpreterFactory.get(settingId); + InterpreterSetting interpreterSetting = interpreterSettingManager.get(settingId); Map infos = interpreterSetting.getInfos(); if (infos != null) { propValue = infos.get(propName); @@ -282,7 +283,7 @@ public Response getMetaInfo(@Context HttpServletRequest req, public Response removeRepository(@PathParam("repoId") String repoId) { logger.info("Remove repository {}", repoId); try { - interpreterFactory.removeRepository(repoId); + interpreterSettingManager.removeRepository(repoId); } catch (Exception e) { logger.error("Exception in InterpreterRestApi while removing repository ", e); return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(), diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index d236214920c..6036ce48596 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -37,7 +37,9 @@ import org.apache.zeppelin.helium.HeliumApplicationFactory; import org.apache.zeppelin.helium.HeliumBundleFactory; import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.NotebookAuthorization; import org.apache.zeppelin.notebook.repo.NotebookRepoSync; @@ -85,6 +87,7 @@ public class ZeppelinServer extends Application { public static NotebookServer notebookWsServer; public static Helium helium; + private final InterpreterSettingManager interpreterSettingManager; private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; private SearchService noteSearchService; @@ -139,14 +142,17 @@ public ZeppelinServer() throws Exception { } this.schedulerFactory = new SchedulerFactory(); + this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, + new InterpreterOption(true)); this.replFactory = new InterpreterFactory(conf, notebookWsServer, - notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated()); + notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(), + interpreterSettingManager); this.notebookRepo = new NotebookRepoSync(conf); this.noteSearchService = new LuceneSearch(); this.notebookAuthorization = NotebookAuthorization.init(conf); this.credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath()); notebook = new Notebook(conf, - notebookRepo, schedulerFactory, replFactory, notebookWsServer, + notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer, noteSearchService, notebookAuthorization, credentials); // to update notebook from application event from remote process. @@ -194,7 +200,7 @@ public static void main(String[] args) throws InterruptedException { LOG.info("Shutting down Zeppelin Server ... "); try { jettyWebServer.stop(); - notebook.getInterpreterFactory().shutdown(); + notebook.getInterpreterSettingManager().shutdown(); notebook.close(); Thread.sleep(3000); } catch (Exception e) { @@ -217,7 +223,7 @@ public static void main(String[] args) throws InterruptedException { } jettyWebServer.join(); - ZeppelinServer.notebook.getInterpreterFactory().close(); + ZeppelinServer.notebook.getInterpreterSettingManager().close(); } private static Server setupJettyServer(ZeppelinConfiguration conf) { @@ -377,7 +383,8 @@ public Set getSingletons() { HeliumRestApi heliumApi = new HeliumRestApi(helium, notebook); singletons.add(heliumApi); - InterpreterRestApi interpreterApi = new InterpreterRestApi(replFactory, notebookWsServer); + InterpreterRestApi interpreterApi = new InterpreterRestApi(interpreterSettingManager, + notebookWsServer); singletons.add(interpreterApi); CredentialRestApi credentialApi = new CredentialRestApi(credentials); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index e692b12fadf..73e0d5ba94e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -47,6 +47,7 @@ import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.helium.HeliumPackage; +import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; @@ -473,7 +474,7 @@ private void broadcastToNoteBindedInterpreter(String interpreterGroupId, Message Notebook notebook = notebook(); List notes = notebook.getAllNotes(); for (Note note : notes) { - List ids = notebook.getInterpreterFactory().getInterpreters(note.getId()); + List ids = notebook.getInterpreterSettingManager().getInterpreters(note.getId()); for (String id : ids) { if (id.equals(interpreterGroupId)) { broadcast(note.getId(), m); @@ -991,7 +992,7 @@ private void createNote(NotebookSocket conn, HashSet userAndRoles, Noteb if (!StringUtils.isEmpty(defaultInterpreterId)) { List interpreterSettingIds = new LinkedList<>(); interpreterSettingIds.add(defaultInterpreterId); - for (String interpreterSettingId : notebook.getInterpreterFactory(). + for (String interpreterSettingId : notebook.getInterpreterSettingManager(). getDefaultInterpreterSettingList()) { if (!interpreterSettingId.equals(defaultInterpreterId)) { interpreterSettingIds.add(interpreterSettingId); @@ -1334,7 +1335,7 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo Note note = notebook.getNote(noteId); if (note != null) { List settings = - notebook.getInterpreterFactory().getInterpreterSettings(note.getId()); + notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { if (setting.getInterpreterGroup(user, note.getId()) == null) { continue; @@ -1376,7 +1377,7 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo // interpreter. for (Note n : notebook.getAllNotes()) { List settings = - notebook.getInterpreterFactory().getInterpreterSettings(note.getId()); + notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { if (setting.getInterpreterGroup(user, n.getId()) == null) { continue; @@ -2197,7 +2198,7 @@ public NotebookEventListener getNotebookInformationListener() { private void sendAllAngularObjects(Note note, String user, NotebookSocket conn) throws IOException { List settings = - notebook().getInterpreterFactory().getInterpreterSettings(note.getId()); + notebook().getInterpreterSettingManager().getInterpreterSettings(note.getId()); if (settings == null || settings.size() == 0) { return; } @@ -2235,7 +2236,7 @@ public void onUpdate(String interpreterGroupId, AngularObject object) { } List intpSettings = - notebook.getInterpreterFactory().getInterpreterSettings(note.getId()); + notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId()); if (intpSettings.isEmpty()) { continue; } @@ -2255,7 +2256,8 @@ public void onRemove(String interpreterGroupId, String name, String noteId, Stri continue; } - List settingIds = notebook.getInterpreterFactory().getInterpreters(note.getId()); + List settingIds = + notebook.getInterpreterSettingManager().getInterpreters(note.getId()); for (String id : settingIds) { if (interpreterGroupId.contains(id)) { broadcast(note.getId(), @@ -2274,21 +2276,25 @@ private void getEditorSetting(NotebookSocket conn, Message fromMessage) throws I String user = fromMessage.principal; Message resp = new Message(OP.EDITOR_SETTING); resp.put("paragraphId", paragraphId); - resp.put("editor", notebook().getInterpreterFactory().getEditorSetting(user, noteId, replName)); + Interpreter interpreter = + notebook().getInterpreterFactory().getInterpreter(user, noteId, replName); + resp.put("editor", notebook().getInterpreterSettingManager(). + getEditorSetting(interpreter, user, noteId, replName)); conn.send(serializeMessage(resp)); return; } private void getInterpreterSettings(NotebookSocket conn, AuthenticationInfo subject) throws IOException { - List availableSettings = notebook().getInterpreterFactory().get(); + List availableSettings = notebook().getInterpreterSettingManager().get(); conn.send(serializeMessage( new Message(OP.INTERPRETER_SETTINGS).put("interpreterSettings", availableSettings))); } @Override public void onMetaInfosReceived(String settingId, Map metaInfos) { - InterpreterSetting interpreterSetting = notebook().getInterpreterFactory().get(settingId); + InterpreterSetting interpreterSetting = + notebook().getInterpreterSettingManager().get(settingId); interpreterSetting.setInfos(metaInfos); } @@ -2342,8 +2348,8 @@ public void onParaInfosReceived(String noteId, String paragraphId, if (note != null) { Paragraph paragraph = note.getParagraph(paragraphId); if (paragraph != null) { - InterpreterSetting setting = notebook().getInterpreterFactory() - .get(interpreterSettingId); + InterpreterSetting setting = notebook().getInterpreterSettingManager() + .get(interpreterSettingId); setting.addNoteToPara(noteId, paragraphId); String label = metaInfos.get("label"); String tooltip = metaInfos.get("tooltip"); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/InterpreterBindingUtils.java b/zeppelin-server/src/main/java/org/apache/zeppelin/utils/InterpreterBindingUtils.java index 9333afd9ef8..94d97fd69fc 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/InterpreterBindingUtils.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/utils/InterpreterBindingUtils.java @@ -37,7 +37,7 @@ public static List getInterpreterBindings(Notebook note setting.getInterpreterInfos(), true)); } - List availableSettings = notebook.getInterpreterFactory().get(); + List availableSettings = notebook.getInterpreterSettingManager().get(); for (InterpreterSetting setting : availableSettings) { boolean selected = false; for (InterpreterSetting selectedSetting : selectedSettings) { diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index fd65ae2bd77..fa61fecaae8 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -188,7 +188,8 @@ private static void start(boolean withAuth) throws Exception { // assume first one is spark InterpreterSetting sparkIntpSetting = null; - for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) { + for(InterpreterSetting intpSetting : + ZeppelinServer.notebook.getInterpreterSettingManager().get()) { if (intpSetting.getName().equals("spark")) { sparkIntpSetting = intpSetting; } @@ -208,7 +209,7 @@ private static void start(boolean withAuth) throws Exception { sparkIntpSetting.setProperties(sparkProperties); pySpark = true; sparkR = true; - ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.getId()); + ZeppelinServer.notebook.getInterpreterSettingManager().restart(sparkIntpSetting.getId()); } else { String sparkHome = getSparkHome(); if (sparkHome != null) { @@ -225,7 +226,7 @@ private static void start(boolean withAuth) throws Exception { sparkR = true; } - ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.getId()); + ZeppelinServer.notebook.getInterpreterSettingManager().restart(sparkIntpSetting.getId()); } } } @@ -292,10 +293,10 @@ private static boolean isActiveSparkHome(File dir) { protected static void shutDown() throws Exception { if (!wasRunning) { // restart interpreter to stop all interpreter processes - List settingList = ZeppelinServer.notebook.getInterpreterFactory() + List settingList = ZeppelinServer.notebook.getInterpreterSettingManager() .getDefaultInterpreterSettingList(); for (String setting : settingList) { - ZeppelinServer.notebook.getInterpreterFactory().restart(setting); + ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting); } if (shiroIni != null) { FileUtils.deleteQuietly(shiroIni); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java index 1214841c615..84cdf66fa6d 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java @@ -80,7 +80,7 @@ public void getAvailableInterpreters() throws IOException { // then assertThat(get, isAllowed()); - assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getAvailableInterpreterSettings().size(), + assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getAvailableInterpreterSettings().size(), body.entrySet().size()); get.releaseConnection(); } @@ -258,7 +258,7 @@ public void testInterpreterRestart() throws IOException, InterruptedException { assertEquals(p.getResult().message().get(0).getData(), getSimulatedMarkdownResult("markdown")); // when: restart interpreter - for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) { + for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId())) { if (setting.getName().equals("md")) { // call restart interpreter API PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), ""); @@ -304,7 +304,7 @@ public void testRestartInterpreterPerNote() throws IOException, InterruptedExcep // when: get md interpreter InterpreterSetting mdIntpSetting = null; - for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) { + for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId())) { if (setting.getName().equals("md")) { mdIntpSetting = setting; break; diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index f0cfab88261..7c790b3e5d3 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -185,7 +185,7 @@ public void sparkRTest() throws IOException { for (InterpreterSetting setting : settings) { if (setting.getName().equals("spark")) { - ZeppelinServer.notebook.getInterpreterFactory().restart(setting.getId()); + ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId()); break; } } @@ -417,7 +417,7 @@ public void pySparkDepLoaderTest() throws IOException { for (InterpreterSetting setting : settings) { if (setting.getName().equals("spark")) { - ZeppelinServer.notebook.getInterpreterFactory().restart(setting.getId()); + ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId()); break; } } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java index 9e64e4069e1..c339140419e 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java @@ -101,7 +101,7 @@ public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() thr // get reference to interpreterGroup InterpreterGroup interpreterGroup = null; - List settings = notebook.getInterpreterFactory().getInterpreterSettings(note1.getId()); + List settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId()); for (InterpreterSetting setting : settings) { if (setting.getName().equals("md")) { interpreterGroup = setting.getInterpreterGroup("anonymous", "sharedProcess"); @@ -374,7 +374,7 @@ public void testCreateNoteWithDefaultInterpreterId() throws IOException { String noteName = "Note with millis " + System.currentTimeMillis(); String defaultInterpreterId = ""; - List settings = notebook.getInterpreterFactory().get(); + List settings = notebook.getInterpreterSettingManager().get(); if (settings.size() > 1) { defaultInterpreterId = settings.get(1).getId(); } @@ -396,7 +396,7 @@ public void testCreateNoteWithDefaultInterpreterId() throws IOException { } if (settings.size() > 1) { - assertEquals(notebook.getInterpreterFactory().getDefaultInterpreterSetting( + assertEquals(notebook.getInterpreterSettingManager().getDefaultInterpreterSetting( createdNote.getId()).getId(), defaultInterpreterId); } notebook.removeNote(createdNote.getId(), anonymous); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index f2340aa97bf..121c6e83d8a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -39,7 +39,18 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.PosixFilePermission; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -72,628 +83,54 @@ import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; -import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; -import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; - /** * Manage interpreters. */ public class InterpreterFactory implements InterpreterGroupFactory { private static final Logger logger = LoggerFactory.getLogger(InterpreterFactory.class); - private static final String SHARED_SESSION = "shared_session"; private Map cleanCl = Collections.synchronizedMap(new HashMap()); private ZeppelinConfiguration conf; - @Deprecated - private String[] interpreterClassList; - private String[] interpreterGroupOrderList; - - /** - * This is only references with default settings, name and properties - * key: InterpreterSetting.name - */ - private final Map interpreterSettingsRef = new HashMap<>(); - - /** - * This is used by creating and running Interpreters - * key: InterpreterSetting.id <- This is becuase backward compatibility - */ - private final Map interpreterSettings = new HashMap<>(); - private Map> interpreterBindings = new HashMap<>(); - private List interpreterRepositories; + private final InterpreterSettingManager interpreterSettingManager; private Gson gson; - private InterpreterOption defaultOption; - private AngularObjectRegistryListener angularObjectRegistryListener; private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; private final ApplicationEventListener appEventListener; - private DependencyResolver depResolver; - private boolean shiroEnabled; private Map env = new HashMap<>(); private Interpreter devInterpreter; - private static final Map DEFAULT_EDITOR = ImmutableMap.of( - "language", (Object) "text", - "editOnDblClick", false); - public InterpreterFactory(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appEventListener, DependencyResolver depResolver, - boolean shiroEnabled) throws InterpreterException, IOException, RepositoryException { - this(conf, new InterpreterOption(true), angularObjectRegistryListener, - remoteInterpreterProcessListener, appEventListener, depResolver, shiroEnabled); - } - - - public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultOption, - AngularObjectRegistryListener angularObjectRegistryListener, - RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appEventListener, DependencyResolver depResolver, - boolean shiroEnabled) throws InterpreterException, IOException, RepositoryException { + boolean shiroEnabled, InterpreterSettingManager interpreterSettingManager) + throws InterpreterException, IOException, RepositoryException { this.conf = conf; - this.defaultOption = defaultOption; this.angularObjectRegistryListener = angularObjectRegistryListener; - this.depResolver = depResolver; - this.interpreterRepositories = depResolver.getRepos(); this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; this.appEventListener = appEventListener; this.shiroEnabled = shiroEnabled; - String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS); - interpreterClassList = replsConf.split(","); - String groupOrder = conf.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER); - interpreterGroupOrderList = groupOrder.split(","); GsonBuilder builder = new GsonBuilder(); builder.setPrettyPrinting(); gson = builder.create(); - init(); + this.interpreterSettingManager = interpreterSettingManager; + //TODO(jl): Fix it not to use InterpreterGroupFactory + interpreterSettingManager.setInterpreterGroupFactory(this); logger.info("shiroEnabled: {}", shiroEnabled); } - private void init() throws InterpreterException, IOException, RepositoryException { - String interpreterJson = conf.getInterpreterJson(); - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - - Path interpretersDir = Paths.get(conf.getInterpreterDir()); - if (Files.exists(interpretersDir)) { - for (Path interpreterDir : Files - .newDirectoryStream(interpretersDir, new DirectoryStream.Filter() { - @Override - public boolean accept(Path entry) throws IOException { - return Files.exists(entry) && Files.isDirectory(entry); - } - })) { - String interpreterDirString = interpreterDir.toString(); - - /** - * Register interpreter by the following ordering - * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/ - * interpreter-setting.json - * 2. Register it from interpreter-setting.json in classpath - * {ZEPPELIN_HOME}/interpreter/{interpreter_name} - * 3. Register it by Interpreter.register - */ - if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) { - if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) { - /* - * TODO(jongyoul) - * - Remove these codes below because of legacy code - * - Support ThreadInterpreter - */ - URLClassLoader ccl = new URLClassLoader( - recursiveBuildLibList(interpreterDir.toFile()), cl); - for (String className : interpreterClassList) { - try { - // Load classes - Class.forName(className, true, ccl); - Set interpreterKeys = Interpreter.registeredInterpreters.keySet(); - for (String interpreterKey : interpreterKeys) { - if (className - .equals(Interpreter.registeredInterpreters.get(interpreterKey) - .getClassName())) { - Interpreter.registeredInterpreters.get(interpreterKey) - .setPath(interpreterDirString); - logger.info("Interpreter " + interpreterKey + " found. class=" + className); - cleanCl.put(interpreterDirString, ccl); - } - } - } catch (Throwable t) { - // nothing to do - } - } - } - } - } - } - - for (RegisteredInterpreter registeredInterpreter : Interpreter.registeredInterpreters - .values()) { - logger - .debug("Registered: {} -> {}. Properties: {}", registeredInterpreter.getInterpreterKey(), - registeredInterpreter.getClassName(), registeredInterpreter.getProperties()); - } - - // RegisteredInterpreters -> interpreterSettingRef - InterpreterInfo interpreterInfo; - for (RegisteredInterpreter r : Interpreter.registeredInterpreters.values()) { - interpreterInfo = - new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(), - r.getEditor()); - add(r.getGroup(), interpreterInfo, r.getProperties(), defaultOption, r.getPath(), - r.getRunner()); - } - - for (String settingId : interpreterSettingsRef.keySet()) { - InterpreterSetting setting = interpreterSettingsRef.get(settingId); - logger.info("InterpreterSettingRef name {}", setting.getName()); - } - - loadFromFile(); - - // if no interpreter settings are loaded, create default set - if (0 == interpreterSettings.size()) { - Map temp = new HashMap<>(); - InterpreterSetting interpreterSetting; - for (InterpreterSetting setting : interpreterSettingsRef.values()) { - interpreterSetting = createFromInterpreterSettingRef(setting); - temp.put(setting.getName(), interpreterSetting); - } - - for (String group : interpreterGroupOrderList) { - if (null != (interpreterSetting = temp.remove(group))) { - interpreterSettings.put(interpreterSetting.getId(), interpreterSetting); - } - } - - for (InterpreterSetting setting : temp.values()) { - interpreterSettings.put(setting.getId(), setting); - } - - saveToFile(); - } - - for (String settingId : interpreterSettings.keySet()) { - InterpreterSetting setting = interpreterSettings.get(settingId); - logger.info("InterpreterSetting group {} : id={}, name={}", setting.getGroup(), settingId, - setting.getName()); - } - } - - private InterpreterSetting createFromInterpreterSettingRef(String name) { - Preconditions.checkNotNull(name, "reference name should be not null"); - InterpreterSetting settingRef = interpreterSettingsRef.get(name); - return createFromInterpreterSettingRef(settingRef); - } - - private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) { - // should return immutable objects - List infos = (null == o.getInterpreterInfos()) ? - new ArrayList() : new ArrayList<>(o.getInterpreterInfos()); - List deps = (null == o.getDependencies()) ? - new ArrayList() : new ArrayList<>(o.getDependencies()); - Properties props = - convertInterpreterProperties((Map) o.getProperties()); - InterpreterOption option = InterpreterOption.fromInterpreterOption(o.getOption()); - - InterpreterSetting setting = new InterpreterSetting(o.getName(), o.getName(), - infos, props, deps, option, o.getPath(), o.getInterpreterRunner()); - setting.setInterpreterGroupFactory(this); - return setting; - } - - private Properties convertInterpreterProperties(Map p) { - Properties properties = new Properties(); - for (String key : p.keySet()) { - properties.put(key, p.get(key).getValue()); - } - return properties; - } - - private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir, - String interpreterJson) throws IOException, RepositoryException { - URL[] urls = recursiveBuildLibList(new File(interpreterDir)); - ClassLoader tempClassLoader = new URLClassLoader(urls, cl); - - Enumeration interpreterSettings = tempClassLoader.getResources(interpreterJson); - if (!interpreterSettings.hasMoreElements()) { - return false; - } - for (URL url : Collections.list(interpreterSettings)) { - try (InputStream inputStream = url.openStream()) { - logger.debug("Reading {} from {}", interpreterJson, url); - List registeredInterpreterList = - getInterpreterListFromJson(inputStream); - registerInterpreters(registeredInterpreterList, interpreterDir); - } - } - return true; - } - - private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson) - throws IOException, RepositoryException { - - Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson); - if (Files.exists(interpreterJsonPath)) { - logger.debug("Reading {}", interpreterJsonPath); - List registeredInterpreterList = - getInterpreterListFromJson(interpreterJsonPath); - registerInterpreters(registeredInterpreterList, interpreterDir); - return true; - } - return false; - } - - private List getInterpreterListFromJson(Path filename) - throws FileNotFoundException { - return getInterpreterListFromJson(new FileInputStream(filename.toFile())); - } - - private List getInterpreterListFromJson(InputStream stream) { - Type registeredInterpreterListType = new TypeToken>() { - }.getType(); - return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType); - } - - private void registerInterpreters(List registeredInterpreters, - String absolutePath) throws IOException, RepositoryException { - - for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) { - InterpreterInfo interpreterInfo = - new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(), - registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor()); - // use defaultOption if it is not specified in interpreter-setting.json - InterpreterOption option = registeredInterpreter.getOption() == null ? defaultOption : - registeredInterpreter.getOption(); - add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(), - option, absolutePath, registeredInterpreter.getRunner()); - } - - } - - private void loadFromFile() throws IOException { - Path settingFile = Paths.get(conf.getInterpreterSettingPath()); - if (!Files.exists(settingFile)) { - // nothing to read - return; - } - InterpreterInfoSaving infoSaving; - try (Reader json = Files.newBufferedReader(settingFile, StandardCharsets.UTF_8)) { - infoSaving = gson.fromJson(json, InterpreterInfoSaving.class); - } - for (String k : infoSaving.interpreterSettings.keySet()) { - InterpreterSetting setting = infoSaving.interpreterSettings.get(k); - List infos = setting.getInterpreterInfos(); - - // Convert json StringMap to Properties - StringMap p = (StringMap) setting.getProperties(); - Properties properties = new Properties(); - for (String key : p.keySet()) { - properties.put(key, p.get(key)); - } - setting.setProperties(properties); - - // Always use separate interpreter process - // While we decided to turn this feature on always (without providing - // enable/disable option on GUI). - // previously created setting should turn this feature on here. - setting.getOption().setRemote(true); - - // Update transient information from InterpreterSettingRef - InterpreterSetting interpreterSettingObject = - interpreterSettingsRef.get(setting.getGroup()); - if (interpreterSettingObject == null) { - logger.warn("can't get InterpreterSetting " + - "Information From loaded Interpreter Setting Ref - {} ", setting.getGroup()); - continue; - } - String depClassPath = interpreterSettingObject.getPath(); - setting.setPath(depClassPath); - - for (InterpreterInfo info : infos) { - if (info.getEditor() == null) { - Map editor = getEditorFromSettingByClassName(interpreterSettingObject, - info.getClassName()); - info.setEditor(editor); - } - } - - setting.setInterpreterGroupFactory(this); - loadInterpreterDependencies(setting); - interpreterSettings.put(k, setting); - } - - this.interpreterBindings = infoSaving.interpreterBindings; - - if (infoSaving.interpreterRepositories != null) { - for (RemoteRepository repo : infoSaving.interpreterRepositories) { - if (!depResolver.getRepos().contains(repo)) { - this.interpreterRepositories.add(repo); - } - } - } - } - - public Map getEditorFromSettingByClassName(InterpreterSetting intpSetting, - String className) { - List intpInfos = intpSetting.getInterpreterInfos(); - for (InterpreterInfo intpInfo : intpInfos) { - - if (className.equals(intpInfo.getClassName())) { - if (intpInfo.getEditor() == null) { - break; - } - return intpInfo.getEditor(); - } - } - return DEFAULT_EDITOR; - } - - private void loadInterpreterDependencies(final InterpreterSetting setting) { - setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES); - setting.setErrorReason(null); - interpreterSettings.put(setting.getId(), setting); - synchronized (interpreterSettings) { - final Thread t = new Thread() { - public void run() { - try { - // dependencies to prevent library conflict - File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + - setting.getId()); - if (localRepoDir.exists()) { - try { - FileUtils.cleanDirectory(localRepoDir); - } catch (FileNotFoundException e) { - logger.info("A file that does not exist cannot be deleted, nothing to worry", e); - } - } - - // load dependencies - List deps = setting.getDependencies(); - if (deps != null) { - for (Dependency d : deps) { - File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); - - if (d.getExclusions() != null) { - depResolver.load(d.getGroupArtifactVersion(), d.getExclusions(), - new File(destDir, setting.getId())); - } else { - depResolver.load(d.getGroupArtifactVersion(), new File(destDir, setting.getId())); - } - } - } - - setting.setStatus(InterpreterSetting.Status.READY); - setting.setErrorReason(null); - } catch (Exception e) { - logger.error(String.format("Error while downloading repos for interpreter group : %s," + - " go to interpreter setting page click on edit and save it again to make " + - "this interpreter work properly. : %s", - setting.getGroup(), e.getLocalizedMessage()), e); - setting.setErrorReason(e.getLocalizedMessage()); - setting.setStatus(InterpreterSetting.Status.ERROR); - } finally { - interpreterSettings.put(setting.getId(), setting); - } - } - }; - t.start(); - } - } - - /** - * Overwrite dependency jar under local-repo/{interpreterId} - * if jar file in original path is changed - */ - private void copyDependenciesFromLocalPath(final InterpreterSetting setting) { - setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES); - interpreterSettings.put(setting.getId(), setting); - synchronized (interpreterSettings) { - final Thread t = new Thread() { - public void run() { - try { - List deps = setting.getDependencies(); - if (deps != null) { - for (Dependency d : deps) { - File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); - - int numSplits = d.getGroupArtifactVersion().split(":").length; - if (!(numSplits >= 3 && numSplits <= 6)) { - depResolver.copyLocalDependency(d.getGroupArtifactVersion(), - new File(destDir, setting.getId())); - } - } - } - setting.setStatus(InterpreterSetting.Status.READY); - } catch (Exception e) { - logger.error(String.format("Error while copying deps for interpreter group : %s," + - " go to interpreter setting page click on edit and save it again to make " + - "this interpreter work properly.", - setting.getGroup()), e); - setting.setErrorReason(e.getLocalizedMessage()); - setting.setStatus(InterpreterSetting.Status.ERROR); - } finally { - interpreterSettings.put(setting.getId(), setting); - } - } - }; - t.start(); - } - } - - void saveToFile() throws IOException { - String jsonString; - - synchronized (interpreterSettings) { - InterpreterInfoSaving info = new InterpreterInfoSaving(); - info.interpreterBindings = interpreterBindings; - info.interpreterSettings = interpreterSettings; - info.interpreterRepositories = interpreterRepositories; - - jsonString = gson.toJson(info); - } - - File settingFile = new File(conf.getInterpreterSettingPath()); - if (!settingFile.exists()) { - settingFile.createNewFile(); - - Set permissions = EnumSet.of(OWNER_READ, OWNER_WRITE); - Files.setPosixFilePermissions(settingFile.toPath(), permissions); - } - - FileOutputStream fos = new FileOutputStream(settingFile, false); - OutputStreamWriter out = new OutputStreamWriter(fos); - out.append(jsonString); - out.close(); - fos.close(); - } - - /** - * Return ordered interpreter setting list. - * The list does not contain more than one setting from the same interpreter class. - * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name - */ - public List getDefaultInterpreterSettingList() { - // this list will contain default interpreter setting list - List defaultSettings = new LinkedList<>(); - - // to ignore the same interpreter group - Map interpreterGroupCheck = new HashMap<>(); - - List sortedSettings = get(); - - for (InterpreterSetting setting : sortedSettings) { - if (defaultSettings.contains(setting.getId())) { - continue; - } - - if (!interpreterGroupCheck.containsKey(setting.getName())) { - defaultSettings.add(setting.getId()); - interpreterGroupCheck.put(setting.getName(), true); - } - } - return defaultSettings; - } - - List getRegisteredInterpreterList() { - return new ArrayList<>(Interpreter.registeredInterpreters.values()); - } - - - private boolean findDefaultInterpreter(List infos) { - for (InterpreterInfo interpreterInfo : infos) { - if (interpreterInfo.isDefaultInterpreter()) { - return true; - } - } - return false; - } - - public InterpreterSetting createNewSetting(String name, String group, - List dependencies, InterpreterOption option, Properties p) throws IOException { - if (name.indexOf(".") >= 0) { - throw new IOException("'.' is invalid for InterpreterSetting name."); - } - InterpreterSetting setting = createFromInterpreterSettingRef(group); - setting.setName(name); - setting.setGroup(group); - setting.appendDependencies(dependencies); - setting.setInterpreterOption(option); - setting.setProperties(p); - setting.setInterpreterGroupFactory(this); - interpreterSettings.put(setting.getId(), setting); - loadInterpreterDependencies(setting); - saveToFile(); - return setting; - } - - private InterpreterSetting add(String group, InterpreterInfo interpreterInfo, - Map interpreterProperties, InterpreterOption option, String path, - InterpreterRunner runner) - throws InterpreterException, IOException, RepositoryException { - ArrayList infos = new ArrayList<>(); - infos.add(interpreterInfo); - return add(group, infos, new ArrayList(), option, interpreterProperties, path, - runner); - } - - /** - * @param group InterpreterSetting reference name - */ - public InterpreterSetting add(String group, ArrayList interpreterInfos, - List dependencies, InterpreterOption option, - Map interpreterProperties, String path, - InterpreterRunner runner) { - Preconditions.checkNotNull(group, "name should not be null"); - Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null"); - Preconditions.checkNotNull(dependencies, "dependencies should not be null"); - Preconditions.checkNotNull(option, "option should not be null"); - Preconditions.checkNotNull(interpreterProperties, "properties should not be null"); - - InterpreterSetting interpreterSetting; - - synchronized (interpreterSettingsRef) { - if (interpreterSettingsRef.containsKey(group)) { - interpreterSetting = interpreterSettingsRef.get(group); - - // Append InterpreterInfo - List infos = interpreterSetting.getInterpreterInfos(); - boolean hasDefaultInterpreter = findDefaultInterpreter(infos); - for (InterpreterInfo interpreterInfo : interpreterInfos) { - if (!infos.contains(interpreterInfo)) { - if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) { - hasDefaultInterpreter = true; - infos.add(0, interpreterInfo); - } else { - infos.add(interpreterInfo); - } - } - } - - // Append dependencies - List dependencyList = interpreterSetting.getDependencies(); - for (Dependency dependency : dependencies) { - if (!dependencyList.contains(dependency)) { - dependencyList.add(dependency); - } - } - - // Append properties - Map properties = - (Map) interpreterSetting.getProperties(); - for (String key : interpreterProperties.keySet()) { - if (!properties.containsKey(key)) { - properties.put(key, interpreterProperties.get(key)); - } - } - - } else { - interpreterSetting = - new InterpreterSetting(group, null, interpreterInfos, interpreterProperties, - dependencies, option, path, runner); - interpreterSettingsRef.put(group, interpreterSetting); - } - } - - if (dependencies.size() > 0) { - loadInterpreterDependencies(interpreterSetting); - } - - interpreterSetting.setInterpreterGroupFactory(this); - return interpreterSetting; - } - /** * @param id interpreterGroup id. Combination of interpreterSettingId + noteId/userId/shared * depends on interpreter mode @@ -723,24 +160,6 @@ public InterpreterGroup createInterpreterGroup(String id, InterpreterOption opti return interpreterGroup; } - public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user, - String noteId) { - InterpreterOption option = interpreterSetting.getOption(); - if (option.isProcess()) { - interpreterSetting.closeAndRemoveInterpreterGroupByNoteId(noteId); - } else if (option.isSession()) { - InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId); - String key = getInterpreterSessionKey(user, noteId, interpreterSetting); - interpreterGroup.close(key); - synchronized (interpreterGroup) { - interpreterGroup.remove(key); - interpreterGroup.notifyAll(); // notify createInterpreterForNote() - } - logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(), - noteId); - } - } - public void createInterpretersForNote(InterpreterSetting interpreterSetting, String user, String noteId, String interpreterSessionKey) { InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId); @@ -805,274 +224,6 @@ public void createInterpretersForNote(InterpreterSetting interpreterSetting, Str } } - - public void remove(String id) throws IOException { - synchronized (interpreterSettings) { - if (interpreterSettings.containsKey(id)) { - InterpreterSetting intp = interpreterSettings.get(id); - intp.closeAndRemoveAllInterpreterGroups(); - - interpreterSettings.remove(id); - for (List settings : interpreterBindings.values()) { - Iterator it = settings.iterator(); - while (it.hasNext()) { - String settingId = it.next(); - if (settingId.equals(id)) { - it.remove(); - } - } - } - saveToFile(); - } - } - - File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id); - FileUtils.deleteDirectory(localRepoDir); - } - - /** - * Get interpreter settings - */ - public List get() { - synchronized (interpreterSettings) { - List orderedSettings = new LinkedList<>(); - - Map> nameInterpreterSettingMap = new HashMap<>(); - for (InterpreterSetting interpreterSetting : interpreterSettings.values()) { - String group = interpreterSetting.getGroup(); - if (!nameInterpreterSettingMap.containsKey(group)) { - nameInterpreterSettingMap.put(group, new ArrayList()); - } - nameInterpreterSettingMap.get(group).add(interpreterSetting); - } - - for (String groupName : interpreterGroupOrderList) { - List interpreterSettingList = - nameInterpreterSettingMap.remove(groupName); - if (null != interpreterSettingList) { - for (InterpreterSetting interpreterSetting : interpreterSettingList) { - orderedSettings.add(interpreterSetting); - } - } - } - - List settings = new ArrayList<>(); - - for (List interpreterSettingList : nameInterpreterSettingMap.values()) { - for (InterpreterSetting interpreterSetting : interpreterSettingList) { - settings.add(interpreterSetting); - } - } - - Collections.sort(settings, new Comparator() { - @Override - public int compare(InterpreterSetting o1, InterpreterSetting o2) { - return o1.getName().compareTo(o2.getName()); - } - }); - - orderedSettings.addAll(settings); - - return orderedSettings; - } - } - - public InterpreterSetting get(String name) { - synchronized (interpreterSettings) { - return interpreterSettings.get(name); - } - } - - private void putNoteInterpreterSettingBinding(String user, String noteId, - List settingList) throws IOException { - List unBindedSettings = new LinkedList<>(); - - synchronized (interpreterSettings) { - List oldSettings = interpreterBindings.get(noteId); - if (oldSettings != null) { - for (String oldSettingId : oldSettings) { - if (!settingList.contains(oldSettingId)) { - unBindedSettings.add(oldSettingId); - } - } - } - interpreterBindings.put(noteId, settingList); - saveToFile(); - - for (String settingId : unBindedSettings) { - InterpreterSetting setting = get(settingId); - removeInterpretersForNote(setting, user, noteId); - } - } - } - - public void removeNoteInterpreterSettingBinding(String user, String noteId) { - synchronized (interpreterSettings) { - List settingIds = (interpreterBindings.containsKey(noteId) ? - interpreterBindings.remove(noteId) : - Collections.emptyList()); - for (String settingId : settingIds) { - this.removeInterpretersForNote(get(settingId), user, noteId); - } - } - } - - private List getNoteInterpreterSettingBinding(String noteId) { - LinkedList bindings = new LinkedList<>(); - synchronized (interpreterSettings) { - List settingIds = interpreterBindings.get(noteId); - if (settingIds != null) { - bindings.addAll(settingIds); - } - } - return bindings; - } - - /** - * Change interpreter property and restart - */ - public void setPropertyAndRestart(String id, InterpreterOption option, Properties properties, - List dependencies) throws IOException { - synchronized (interpreterSettings) { - InterpreterSetting intpSetting = interpreterSettings.get(id); - if (intpSetting != null) { - try { - stopJobAllInterpreter(intpSetting); - - intpSetting.closeAndRemoveAllInterpreterGroups(); - intpSetting.setOption(option); - intpSetting.setProperties(properties); - intpSetting.setDependencies(dependencies); - loadInterpreterDependencies(intpSetting); - - saveToFile(); - } catch (Exception e) { - throw e; - } finally { - loadFromFile(); - } - } else { - throw new InterpreterException("Interpreter setting id " + id + " not found"); - } - } - } - - private boolean noteIdIsExist(String noteId) { - return noteId == null ? false : true; - } - - public void restart(String settingId, String noteId, String user) { - InterpreterSetting intpSetting = interpreterSettings.get(settingId); - Preconditions.checkNotNull(intpSetting); - - // restart interpreter setting in note page - if (noteIdIsExist(noteId) && intpSetting.getOption().isProcess()) { - intpSetting.closeAndRemoveInterpreterGroupByNoteId(noteId); - return; - } else { - // restart interpreter setting in interpreter setting page - restart(settingId, user); - } - - - } - - public void restart(String id, String user) { - synchronized (interpreterSettings) { - InterpreterSetting intpSetting = interpreterSettings.get(id); - // Check if dependency in specified path is changed - // If it did, overwrite old dependency jar with new one - if (intpSetting != null) { - //clean up metaInfos - intpSetting.setInfos(null); - copyDependenciesFromLocalPath(intpSetting); - - stopJobAllInterpreter(intpSetting); - if (user.equals("anonymous")) { - intpSetting.closeAndRemoveAllInterpreterGroups(); - } else { - intpSetting.closeAndRemoveInterpreterGroupByUser(user); - } - - } else { - throw new InterpreterException("Interpreter setting id " + id + " not found"); - } - } - } - - public void restart(String id) { - restart(id, "anonymous"); - } - - private void stopJobAllInterpreter(InterpreterSetting intpSetting) { - if (intpSetting != null) { - for (InterpreterGroup intpGroup : intpSetting.getAllInterpreterGroups()) { - for (List interpreters : intpGroup.values()) { - for (Interpreter intp : interpreters) { - for (Job job : intp.getScheduler().getJobsRunning()) { - job.abort(); - job.setStatus(Status.ABORT); - logger.info("Job " + job.getJobName() + " aborted "); - } - for (Job job : intp.getScheduler().getJobsWaiting()) { - job.abort(); - job.setStatus(Status.ABORT); - logger.info("Job " + job.getJobName() + " aborted "); - } - } - } - } - } - } - - public void close() { - List closeThreads = new LinkedList<>(); - synchronized (interpreterSettings) { - Collection intpSettings = interpreterSettings.values(); - for (final InterpreterSetting intpSetting : intpSettings) { - Thread t = new Thread() { - public void run() { - intpSetting.closeAndRemoveAllInterpreterGroups(); - } - }; - t.start(); - closeThreads.add(t); - } - } - - for (Thread t : closeThreads) { - try { - t.join(); - } catch (InterruptedException e) { - logger.error("Can't close interpreterGroup", e); - } - } - } - - public void shutdown() { - List closeThreads = new LinkedList<>(); - synchronized (interpreterSettings) { - Collection intpSettings = interpreterSettings.values(); - for (final InterpreterSetting intpSetting : intpSettings) { - Thread t = new Thread() { - public void run() { - intpSetting.shutdownAndRemoveAllInterpreterGroups(); - } - }; - t.start(); - closeThreads.add(t); - } - } - - for (Thread t : closeThreads) { - try { - t.join(); - } catch (InterruptedException e) { - logger.error("Can't close interpreterGroup", e); - } - } - } - private Interpreter createRepl(String dirName, String className, Properties property) throws InterpreterException { logger.info("Create repl {} from {}", className, dirName); @@ -1172,76 +323,12 @@ Interpreter createRemoteRepl(String interpreterPath, String interpreterSessionKe return new LazyOpenInterpreter(remoteInterpreter); } - /** - * map interpreter ids into noteId - * - * @param noteId note id - * @param ids InterpreterSetting id list - */ - public void setInterpreters(String user, String noteId, List ids) throws IOException { - putNoteInterpreterSettingBinding(user, noteId, ids); - } - - public List getInterpreters(String noteId) { - return getNoteInterpreterSettingBinding(noteId); - } - - public List getInterpreterSettings(String noteId) { - List interpreterSettingIds = getNoteInterpreterSettingBinding(noteId); - LinkedList settings = new LinkedList<>(); - - Iterator iter = interpreterSettingIds.iterator(); - while (iter.hasNext()) { - String id = iter.next(); - InterpreterSetting setting = get(id); - if (setting == null) { - // interpreter setting is removed from factory. remove id from here, too - iter.remove(); - } else { - settings.add(setting); - } - } - return settings; - } - - public void closeNote(String user, String noteId) { - // close interpreters in this note session - List settings = getInterpreterSettings(noteId); - if (settings == null || settings.size() == 0) { - return; - } - - logger.info("closeNote: {}", noteId); - for (InterpreterSetting setting : settings) { - removeInterpretersForNote(setting, user, noteId); - } - } - - private String getInterpreterSessionKey(String user, String noteId, InterpreterSetting setting) { - InterpreterOption option = setting.getOption(); - String key; - if (option.isExistingProcess()) { - key = Constants.EXISTING_PROCESS; - } else if (option.perNoteScoped() && option.perUserScoped()) { - key = user + ":" + noteId; - } else if (option.perUserScoped()) { - key = user; - } else if (option.perNoteScoped()) { - key = noteId; - } else { - key = SHARED_SESSION; - } - - logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " + - "{}", key, noteId, user, setting.getName()); - return key; - } - private List createOrGetInterpreterList(String user, String noteId, InterpreterSetting setting) { InterpreterGroup interpreterGroup = setting.getInterpreterGroup(user, noteId); synchronized (interpreterGroup) { - String interpreterSessionKey = getInterpreterSessionKey(user, noteId, setting); + String interpreterSessionKey = + interpreterSettingManager.getInterpreterSessionKey(user, noteId, setting); if (!interpreterGroup.containsKey(interpreterSessionKey)) { createInterpretersForNote(setting, user, noteId, interpreterSessionKey); } @@ -1249,18 +336,6 @@ private List createOrGetInterpreterList(String user, String noteId, } } - private InterpreterSetting getDefaultInterpreterSetting(List settings) { - if (settings == null || settings.isEmpty()) { - return null; - } - return settings.get(0); - } - - public InterpreterSetting getDefaultInterpreterSetting(String noteId) { - return getDefaultInterpreterSetting(getInterpreterSettings(noteId)); - } - - private InterpreterSetting getInterpreterSettingByGroup(List settings, String group) { Preconditions.checkNotNull(group, "group should be not null"); @@ -1305,7 +380,7 @@ private Interpreter getInterpreter(String user, String noteId, InterpreterSettin } public Interpreter getInterpreter(String user, String noteId, String replName) { - List settings = getInterpreterSettings(noteId); + List settings = interpreterSettingManager.getInterpreterSettings(noteId); InterpreterSetting setting; Interpreter interpreter; @@ -1316,7 +391,8 @@ public Interpreter getInterpreter(String user, String noteId, String replName) { if (replName == null || replName.trim().length() == 0) { // get default settings (first available) // TODO(jl): Fix it in case of returning null - InterpreterSetting defaultSettings = getDefaultInterpreterSetting(settings); + InterpreterSetting defaultSettings = interpreterSettingManager + .getDefaultInterpreterSetting(settings); return createOrGetInterpreterList(user, noteId, defaultSettings).get(0); } @@ -1343,7 +419,7 @@ public Interpreter getInterpreter(String user, String noteId, String replName) { // first assume replName is 'name' of interpreter. ('groupName' is ommitted) // search 'name' from first (default) interpreter group // TODO(jl): Handle with noteId to support defaultInterpreter per note. - setting = getDefaultInterpreterSetting(settings); + setting = interpreterSettingManager.getDefaultInterpreterSetting(settings); interpreter = getInterpreter(user, noteId, setting, replName); @@ -1376,44 +452,6 @@ public Interpreter getInterpreter(String user, String noteId, String replName) { return null; } - public Map getAvailableInterpreterSettings() { - return interpreterSettingsRef; - } - - private URL[] recursiveBuildLibList(File path) throws MalformedURLException { - URL[] urls = new URL[0]; - if (path == null || !path.exists()) { - return urls; - } else if (path.getName().startsWith(".")) { - return urls; - } else if (path.isDirectory()) { - File[] files = path.listFiles(); - if (files != null) { - for (File f : files) { - urls = (URL[]) ArrayUtils.addAll(urls, recursiveBuildLibList(f)); - } - } - return urls; - } else { - return new URL[]{path.toURI().toURL()}; - } - } - - public List getRepositories() { - return this.interpreterRepositories; - } - - public void addRepository(String id, String url, boolean snapshot, Authentication auth, - Proxy proxy) throws IOException { - depResolver.addRepo(id, url, snapshot, auth, proxy); - saveToFile(); - } - - public void removeRepository(String id) throws IOException { - depResolver.delRepo(id); - saveToFile(); - } - public Map getEnv() { return env; } @@ -1422,31 +460,5 @@ public void setEnv(Map env) { this.env = env; } - public Map getEditorSetting(String user, String noteId, String replName) { - Interpreter intp = getInterpreter(user, noteId, replName); - Map editor = DEFAULT_EDITOR; - String group = StringUtils.EMPTY; - try { - String defaultSettingName = getDefaultInterpreterSetting(noteId).getName(); - List intpSettings = getInterpreterSettings(noteId); - for (InterpreterSetting intpSetting : intpSettings) { - String[] replNameSplit = replName.split("\\."); - if (replNameSplit.length == 2) { - group = replNameSplit[0]; - } - // when replName is 'name' of interpreter - if (defaultSettingName.equals(intpSetting.getName())) { - editor = getEditorFromSettingByClassName(intpSetting, intp.getClassName()); - } - // when replName is 'alias name' of interpreter or 'group' of interpreter - if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) { - editor = getEditorFromSettingByClassName(intpSetting, intp.getClassName()); - break; - } - } - } catch (NullPointerException e) { - logger.warn("Couldn't get interpreter editor setting"); - } - return editor; - } + } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java new file mode 100644 index 00000000000..b141e6b27a4 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -0,0 +1,1125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter; + +import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; +import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.internal.StringMap; +import com.google.gson.reflect.TypeToken; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.Dependency; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; +import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.Job.Status; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.sonatype.aether.RepositoryException; +import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.Proxy; +import org.sonatype.aether.repository.RemoteRepository; + +/** + * TBD + */ +public class InterpreterSettingManager { + + private static final Logger logger = LoggerFactory.getLogger(InterpreterSettingManager.class); + private static final String SHARED_SESSION = "shared_session"; + private static final Map DEFAULT_EDITOR = ImmutableMap.of( + "language", (Object) "text", + "editOnDblClick", false); + + private final ZeppelinConfiguration zeppelinConfiguration; + private final Path interpreterDirPath; + private final Path interpreterBindingPath; + + /** + * This is only references with default settings, name and properties + * key: InterpreterSetting.name + */ + private final Map interpreterSettingsRef; + /** + * This is used by creating and running Interpreters + * key: InterpreterSetting.id <- This is becuase backward compatibility + */ + private final Map interpreterSettings; + private final Map> interpreterBindings; + + private final DependencyResolver dependencyResolver; + private final List interpreterRepositories; + + private final InterpreterOption defaultOption; + + private final Map cleanCl; + + @Deprecated + private String[] interpreterClassList; + private String[] interpreterGroupOrderList; + private InterpreterGroupFactory interpreterGroupFactory; + + private final Gson gson; + + public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration, + DependencyResolver dependencyResolver, InterpreterOption interpreterOption) + throws IOException, RepositoryException { + this.zeppelinConfiguration = zeppelinConfiguration; + this.interpreterDirPath = Paths.get(zeppelinConfiguration.getInterpreterDir()); + logger.debug("InterpreterRootPath: {}", interpreterDirPath); + this.interpreterBindingPath = Paths.get(zeppelinConfiguration.getInterpreterSettingPath()); + logger.debug("InterpreterBindingPath: {}", interpreterBindingPath); + + this.interpreterSettingsRef = Maps.newConcurrentMap(); + this.interpreterSettings = Maps.newConcurrentMap(); + this.interpreterBindings = Maps.newConcurrentMap(); + + this.dependencyResolver = dependencyResolver; + this.interpreterRepositories = dependencyResolver.getRepos(); + + this.defaultOption = interpreterOption; + + this.cleanCl = Collections.synchronizedMap(new HashMap()); + + String replsConf = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETERS); + this.interpreterClassList = replsConf.split(","); + String groupOrder = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER); + this.interpreterGroupOrderList = groupOrder.split(","); + + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.setPrettyPrinting(); + this.gson = gsonBuilder.create(); + + init(); + } + + private void loadFromFile() { + if (!Files.exists(interpreterBindingPath)) { + // nothing to read + return; + } + InterpreterInfoSaving infoSaving; + try (BufferedReader json = + Files.newBufferedReader(interpreterBindingPath, StandardCharsets.UTF_8)) { + infoSaving = gson.fromJson(json, InterpreterInfoSaving.class); + + for (String k : infoSaving.interpreterSettings.keySet()) { + InterpreterSetting setting = infoSaving.interpreterSettings.get(k); + List infos = setting.getInterpreterInfos(); + + // Convert json StringMap to Properties + StringMap p = (StringMap) setting.getProperties(); + Properties properties = new Properties(); + for (String key : p.keySet()) { + properties.put(key, p.get(key)); + } + setting.setProperties(properties); + + // Always use separate interpreter process + // While we decided to turn this feature on always (without providing + // enable/disable option on GUI). + // previously created setting should turn this feature on here. + setting.getOption().setRemote(true); + + // Update transient information from InterpreterSettingRef + InterpreterSetting interpreterSettingObject = + interpreterSettingsRef.get(setting.getGroup()); + if (interpreterSettingObject == null) { + logger.warn("can't get InterpreterSetting " + + "Information From loaded Interpreter Setting Ref - {} ", setting.getGroup()); + continue; + } + String depClassPath = interpreterSettingObject.getPath(); + setting.setPath(depClassPath); + + for (InterpreterInfo info : infos) { + if (info.getEditor() == null) { + Map editor = getEditorFromSettingByClassName(interpreterSettingObject, + info.getClassName()); + info.setEditor(editor); + } + } + + setting.setInterpreterGroupFactory(interpreterGroupFactory); + + loadInterpreterDependencies(setting); + interpreterSettings.put(k, setting); + } + + interpreterBindings.putAll(infoSaving.interpreterBindings); + + if (infoSaving.interpreterRepositories != null) { + for (RemoteRepository repo : infoSaving.interpreterRepositories) { + if (!dependencyResolver.getRepos().contains(repo)) { + this.interpreterRepositories.add(repo); + } + } + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void saveToFile() throws IOException { + String jsonString; + + synchronized (interpreterSettings) { + InterpreterInfoSaving info = new InterpreterInfoSaving(); + info.interpreterBindings = interpreterBindings; + info.interpreterSettings = interpreterSettings; + info.interpreterRepositories = interpreterRepositories; + + jsonString = gson.toJson(info); + } + + if (!Files.exists(interpreterBindingPath)) { + Files.createFile(interpreterBindingPath); + + Set permissions = EnumSet.of(OWNER_READ, OWNER_WRITE); + Files.setPosixFilePermissions(interpreterBindingPath, permissions); + } + + FileOutputStream fos = new FileOutputStream(interpreterBindingPath.toFile(), false); + OutputStreamWriter out = new OutputStreamWriter(fos); + out.append(jsonString); + out.close(); + fos.close(); + } + + //TODO(jl): Fix it to remove InterpreterGroupFactory + public void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) { + for (InterpreterSetting setting : interpreterSettings.values()) { + setting.setInterpreterGroupFactory(interpreterGroupFactory); + } + this.interpreterGroupFactory = interpreterGroupFactory; + } + + private void init() throws InterpreterException, IOException, RepositoryException { + String interpreterJson = zeppelinConfiguration.getInterpreterJson(); + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + + if (Files.exists(interpreterDirPath)) { + for (Path interpreterDir : Files + .newDirectoryStream(interpreterDirPath, new Filter() { + @Override + public boolean accept(Path entry) throws IOException { + return Files.exists(entry) && Files.isDirectory(entry); + } + })) { + String interpreterDirString = interpreterDir.toString(); + + /** + * Register interpreter by the following ordering + * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/ + * interpreter-setting.json + * 2. Register it from interpreter-setting.json in classpath + * {ZEPPELIN_HOME}/interpreter/{interpreter_name} + * 3. Register it by Interpreter.register + */ + if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) { + if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) { + /* + * TODO(jongyoul) + * - Remove these codes below because of legacy code + * - Support ThreadInterpreter + */ + URLClassLoader ccl = new URLClassLoader( + recursiveBuildLibList(interpreterDir.toFile()), cl); + for (String className : interpreterClassList) { + try { + // Load classes + Class.forName(className, true, ccl); + Set interpreterKeys = Interpreter.registeredInterpreters.keySet(); + for (String interpreterKey : interpreterKeys) { + if (className + .equals(Interpreter.registeredInterpreters.get(interpreterKey) + .getClassName())) { + Interpreter.registeredInterpreters.get(interpreterKey) + .setPath(interpreterDirString); + logger.info("Interpreter " + interpreterKey + " found. class=" + className); + cleanCl.put(interpreterDirString, ccl); + } + } + } catch (Throwable t) { + // nothing to do + } + } + } + } + } + } + + for (RegisteredInterpreter registeredInterpreter : Interpreter.registeredInterpreters + .values()) { + logger + .debug("Registered: {} -> {}. Properties: {}", registeredInterpreter.getInterpreterKey(), + registeredInterpreter.getClassName(), registeredInterpreter.getProperties()); + } + + // RegisteredInterpreters -> interpreterSettingRef + InterpreterInfo interpreterInfo; + for (RegisteredInterpreter r : Interpreter.registeredInterpreters.values()) { + interpreterInfo = + new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(), + r.getEditor()); + add(r.getGroup(), interpreterInfo, r.getProperties(), defaultOption, r.getPath(), + r.getRunner()); + } + + for (String settingId : interpreterSettingsRef.keySet()) { + InterpreterSetting setting = interpreterSettingsRef.get(settingId); + logger.info("InterpreterSettingRef name {}", setting.getName()); + } + + loadFromFile(); + + // if no interpreter settings are loaded, create default set + if (0 == interpreterSettings.size()) { + Map temp = new HashMap<>(); + InterpreterSetting interpreterSetting; + for (InterpreterSetting setting : interpreterSettingsRef.values()) { + interpreterSetting = createFromInterpreterSettingRef(setting); + temp.put(setting.getName(), interpreterSetting); + } + + for (String group : interpreterGroupOrderList) { + if (null != (interpreterSetting = temp.remove(group))) { + interpreterSettings.put(interpreterSetting.getId(), interpreterSetting); + } + } + + for (InterpreterSetting setting : temp.values()) { + interpreterSettings.put(setting.getId(), setting); + } + + saveToFile(); + } + + for (String settingId : interpreterSettings.keySet()) { + InterpreterSetting setting = interpreterSettings.get(settingId); + logger.info("InterpreterSetting group {} : id={}, name={}", setting.getGroup(), settingId, + setting.getName()); + } + } + + private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir, + String interpreterJson) throws IOException, RepositoryException { + URL[] urls = recursiveBuildLibList(new File(interpreterDir)); + ClassLoader tempClassLoader = new URLClassLoader(urls, cl); + + Enumeration interpreterSettings = tempClassLoader.getResources(interpreterJson); + if (!interpreterSettings.hasMoreElements()) { + return false; + } + for (URL url : Collections.list(interpreterSettings)) { + try (InputStream inputStream = url.openStream()) { + logger.debug("Reading {} from {}", interpreterJson, url); + List registeredInterpreterList = + getInterpreterListFromJson(inputStream); + registerInterpreters(registeredInterpreterList, interpreterDir); + } + } + return true; + } + + private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson) + throws IOException, RepositoryException { + + Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson); + if (Files.exists(interpreterJsonPath)) { + logger.debug("Reading {}", interpreterJsonPath); + List registeredInterpreterList = + getInterpreterListFromJson(interpreterJsonPath); + registerInterpreters(registeredInterpreterList, interpreterDir); + return true; + } + return false; + } + + private List getInterpreterListFromJson(Path filename) + throws FileNotFoundException { + return getInterpreterListFromJson(new FileInputStream(filename.toFile())); + } + + private List getInterpreterListFromJson(InputStream stream) { + Type registeredInterpreterListType = new TypeToken>() { + }.getType(); + return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType); + } + + private void registerInterpreters(List registeredInterpreters, + String absolutePath) throws IOException, RepositoryException { + + for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) { + InterpreterInfo interpreterInfo = + new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(), + registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor()); + // use defaultOption if it is not specified in interpreter-setting.json + InterpreterOption option = registeredInterpreter.getOption() == null ? defaultOption : + registeredInterpreter.getOption(); + add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(), + option, absolutePath, registeredInterpreter.getRunner()); + } + + } + + public InterpreterSetting getDefaultInterpreterSetting(List settings) { + if (settings == null || settings.isEmpty()) { + return null; + } + return settings.get(0); + } + + public InterpreterSetting getDefaultInterpreterSetting(String noteId) { + return getDefaultInterpreterSetting(getInterpreterSettings(noteId)); + } + + public List getInterpreterSettings(String noteId) { + List interpreterSettingIds = getNoteInterpreterSettingBinding(noteId); + LinkedList settings = new LinkedList<>(); + + Iterator iter = interpreterSettingIds.iterator(); + while (iter.hasNext()) { + String id = iter.next(); + InterpreterSetting setting = get(id); + if (setting == null) { + // interpreter setting is removed from factory. remove id from here, too + iter.remove(); + } else { + settings.add(setting); + } + } + return settings; + } + + private List getNoteInterpreterSettingBinding(String noteId) { + LinkedList bindings = new LinkedList<>(); + synchronized (interpreterSettings) { + List settingIds = interpreterBindings.get(noteId); + if (settingIds != null) { + bindings.addAll(settingIds); + } + } + return bindings; + } + + private InterpreterSetting createFromInterpreterSettingRef(String name) { + Preconditions.checkNotNull(name, "reference name should be not null"); + InterpreterSetting settingRef = interpreterSettingsRef.get(name); + return createFromInterpreterSettingRef(settingRef); + } + + private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) { + // should return immutable objects + List infos = (null == o.getInterpreterInfos()) ? + new ArrayList() : new ArrayList<>(o.getInterpreterInfos()); + List deps = (null == o.getDependencies()) ? + new ArrayList() : new ArrayList<>(o.getDependencies()); + Properties props = + convertInterpreterProperties((Map) o.getProperties()); + InterpreterOption option = InterpreterOption.fromInterpreterOption(o.getOption()); + + InterpreterSetting setting = new InterpreterSetting(o.getName(), o.getName(), + infos, props, deps, option, o.getPath(), o.getInterpreterRunner()); + setting.setInterpreterGroupFactory(interpreterGroupFactory); + return setting; + } + + private Properties convertInterpreterProperties(Map p) { + Properties properties = new Properties(); + for (String key : p.keySet()) { + properties.put(key, p.get(key).getValue()); + } + return properties; + } + + public Map getEditorSetting(Interpreter interpreter, String user, String noteId, + String replName) { + Map editor = DEFAULT_EDITOR; + String group = StringUtils.EMPTY; + try { + String defaultSettingName = getDefaultInterpreterSetting(noteId).getName(); + List intpSettings = getInterpreterSettings(noteId); + for (InterpreterSetting intpSetting : intpSettings) { + String[] replNameSplit = replName.split("\\."); + if (replNameSplit.length == 2) { + group = replNameSplit[0]; + } + // when replName is 'name' of interpreter + if (defaultSettingName.equals(intpSetting.getName())) { + editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName()); + } + // when replName is 'alias name' of interpreter or 'group' of interpreter + if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) { + editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName()); + break; + } + } + } catch (NullPointerException e) { + logger.warn("Couldn't get interpreter editor setting"); + } + return editor; + } + + public Map getEditorFromSettingByClassName(InterpreterSetting intpSetting, + String className) { + List intpInfos = intpSetting.getInterpreterInfos(); + for (InterpreterInfo intpInfo : intpInfos) { + + if (className.equals(intpInfo.getClassName())) { + if (intpInfo.getEditor() == null) { + break; + } + return intpInfo.getEditor(); + } + } + return DEFAULT_EDITOR; + } + + private void loadInterpreterDependencies(final InterpreterSetting setting) { + setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES); + setting.setErrorReason(null); + interpreterSettings.put(setting.getId(), setting); + synchronized (interpreterSettings) { + final Thread t = new Thread() { + public void run() { + try { + // dependencies to prevent library conflict + File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + + setting.getId()); + if (localRepoDir.exists()) { + try { + FileUtils.cleanDirectory(localRepoDir); + } catch (FileNotFoundException e) { + logger.info("A file that does not exist cannot be deleted, nothing to worry", e); + } + } + + // load dependencies + List deps = setting.getDependencies(); + if (deps != null) { + for (Dependency d : deps) { + File destDir = new File( + zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); + + if (d.getExclusions() != null) { + dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(), + new File(destDir, setting.getId())); + } else { + dependencyResolver + .load(d.getGroupArtifactVersion(), new File(destDir, setting.getId())); + } + } + } + + setting.setStatus(InterpreterSetting.Status.READY); + setting.setErrorReason(null); + } catch (Exception e) { + logger.error(String.format("Error while downloading repos for interpreter group : %s," + + " go to interpreter setting page click on edit and save it again to make " + + "this interpreter work properly. : %s", + setting.getGroup(), e.getLocalizedMessage()), e); + setting.setErrorReason(e.getLocalizedMessage()); + setting.setStatus(InterpreterSetting.Status.ERROR); + } finally { + interpreterSettings.put(setting.getId(), setting); + } + } + }; + t.start(); + } + } + + /** + * Overwrite dependency jar under local-repo/{interpreterId} + * if jar file in original path is changed + */ + private void copyDependenciesFromLocalPath(final InterpreterSetting setting) { + setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES); + interpreterSettings.put(setting.getId(), setting); + synchronized (interpreterSettings) { + final Thread t = new Thread() { + public void run() { + try { + List deps = setting.getDependencies(); + if (deps != null) { + for (Dependency d : deps) { + File destDir = new File( + zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); + + int numSplits = d.getGroupArtifactVersion().split(":").length; + if (!(numSplits >= 3 && numSplits <= 6)) { + dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(), + new File(destDir, setting.getId())); + } + } + } + setting.setStatus(InterpreterSetting.Status.READY); + } catch (Exception e) { + logger.error(String.format("Error while copying deps for interpreter group : %s," + + " go to interpreter setting page click on edit and save it again to make " + + "this interpreter work properly.", + setting.getGroup()), e); + setting.setErrorReason(e.getLocalizedMessage()); + setting.setStatus(InterpreterSetting.Status.ERROR); + } finally { + interpreterSettings.put(setting.getId(), setting); + } + } + }; + t.start(); + } + } + + /** + * Return ordered interpreter setting list. + * The list does not contain more than one setting from the same interpreter class. + * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name + */ + public List getDefaultInterpreterSettingList() { + // this list will contain default interpreter setting list + List defaultSettings = new LinkedList<>(); + + // to ignore the same interpreter group + Map interpreterGroupCheck = new HashMap<>(); + + List sortedSettings = get(); + + for (InterpreterSetting setting : sortedSettings) { + if (defaultSettings.contains(setting.getId())) { + continue; + } + + if (!interpreterGroupCheck.containsKey(setting.getName())) { + defaultSettings.add(setting.getId()); + interpreterGroupCheck.put(setting.getName(), true); + } + } + return defaultSettings; + } + + List getRegisteredInterpreterList() { + return new ArrayList<>(Interpreter.registeredInterpreters.values()); + } + + + private boolean findDefaultInterpreter(List infos) { + for (InterpreterInfo interpreterInfo : infos) { + if (interpreterInfo.isDefaultInterpreter()) { + return true; + } + } + return false; + } + + public InterpreterSetting createNewSetting(String name, String group, + List dependencies, InterpreterOption option, Properties p) throws IOException { + if (name.indexOf(".") >= 0) { + throw new IOException("'.' is invalid for InterpreterSetting name."); + } + InterpreterSetting setting = createFromInterpreterSettingRef(group); + setting.setName(name); + setting.setGroup(group); + setting.appendDependencies(dependencies); + setting.setInterpreterOption(option); + setting.setProperties(p); + setting.setInterpreterGroupFactory(interpreterGroupFactory); + interpreterSettings.put(setting.getId(), setting); + loadInterpreterDependencies(setting); + saveToFile(); + return setting; + } + + private InterpreterSetting add(String group, InterpreterInfo interpreterInfo, + Map interpreterProperties, InterpreterOption option, String path, + InterpreterRunner runner) + throws InterpreterException, IOException, RepositoryException { + ArrayList infos = new ArrayList<>(); + infos.add(interpreterInfo); + return add(group, infos, new ArrayList(), option, interpreterProperties, path, + runner); + } + + /** + * @param group InterpreterSetting reference name + */ + public InterpreterSetting add(String group, ArrayList interpreterInfos, + List dependencies, InterpreterOption option, + Map interpreterProperties, String path, + InterpreterRunner runner) { + Preconditions.checkNotNull(group, "name should not be null"); + Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null"); + Preconditions.checkNotNull(dependencies, "dependencies should not be null"); + Preconditions.checkNotNull(option, "option should not be null"); + Preconditions.checkNotNull(interpreterProperties, "properties should not be null"); + + InterpreterSetting interpreterSetting; + + synchronized (interpreterSettingsRef) { + if (interpreterSettingsRef.containsKey(group)) { + interpreterSetting = interpreterSettingsRef.get(group); + + // Append InterpreterInfo + List infos = interpreterSetting.getInterpreterInfos(); + boolean hasDefaultInterpreter = findDefaultInterpreter(infos); + for (InterpreterInfo interpreterInfo : interpreterInfos) { + if (!infos.contains(interpreterInfo)) { + if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) { + hasDefaultInterpreter = true; + infos.add(0, interpreterInfo); + } else { + infos.add(interpreterInfo); + } + } + } + + // Append dependencies + List dependencyList = interpreterSetting.getDependencies(); + for (Dependency dependency : dependencies) { + if (!dependencyList.contains(dependency)) { + dependencyList.add(dependency); + } + } + + // Append properties + Map properties = + (Map) interpreterSetting.getProperties(); + for (String key : interpreterProperties.keySet()) { + if (!properties.containsKey(key)) { + properties.put(key, interpreterProperties.get(key)); + } + } + + } else { + interpreterSetting = + new InterpreterSetting(group, null, interpreterInfos, interpreterProperties, + dependencies, option, path, runner); + interpreterSettingsRef.put(group, interpreterSetting); + } + } + + if (dependencies.size() > 0) { + loadInterpreterDependencies(interpreterSetting); + } + + interpreterSetting.setInterpreterGroupFactory(interpreterGroupFactory); + return interpreterSetting; + } + + /** + * map interpreter ids into noteId + * + * @param noteId note id + * @param ids InterpreterSetting id list + */ + public void setInterpreters(String user, String noteId, List ids) throws IOException { + putNoteInterpreterSettingBinding(user, noteId, ids); + } + + private void putNoteInterpreterSettingBinding(String user, String noteId, + List settingList) throws IOException { + List unBindedSettings = new LinkedList<>(); + + synchronized (interpreterSettings) { + List oldSettings = interpreterBindings.get(noteId); + if (oldSettings != null) { + for (String oldSettingId : oldSettings) { + if (!settingList.contains(oldSettingId)) { + unBindedSettings.add(oldSettingId); + } + } + } + interpreterBindings.put(noteId, settingList); + saveToFile(); + + for (String settingId : unBindedSettings) { + InterpreterSetting setting = get(settingId); + removeInterpretersForNote(setting, user, noteId); + } + } + } + + public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user, + String noteId) { + InterpreterOption option = interpreterSetting.getOption(); + if (option.isProcess()) { + interpreterSetting.closeAndRemoveInterpreterGroupByNoteId(noteId); + } else if (option.isSession()) { + InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId); + String key = getInterpreterSessionKey(user, noteId, interpreterSetting); + interpreterGroup.close(key); + synchronized (interpreterGroup) { + interpreterGroup.remove(key); + interpreterGroup.notifyAll(); // notify createInterpreterForNote() + } + logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(), + noteId); + } + } + + public String getInterpreterSessionKey(String user, String noteId, InterpreterSetting setting) { + InterpreterOption option = setting.getOption(); + String key; + if (option.isExistingProcess()) { + key = Constants.EXISTING_PROCESS; + } else if (option.perNoteScoped() && option.perUserScoped()) { + key = user + ":" + noteId; + } else if (option.perUserScoped()) { + key = user; + } else if (option.perNoteScoped()) { + key = noteId; + } else { + key = SHARED_SESSION; + } + + logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " + + "{}", key, noteId, user, setting.getName()); + return key; + } + + + public List getInterpreters(String noteId) { + return getNoteInterpreterSettingBinding(noteId); + } + + public void closeNote(String user, String noteId) { + // close interpreters in this note session + List settings = getInterpreterSettings(noteId); + if (settings == null || settings.size() == 0) { + return; + } + + logger.info("closeNote: {}", noteId); + for (InterpreterSetting setting : settings) { + removeInterpretersForNote(setting, user, noteId); + } + } + + public Map getAvailableInterpreterSettings() { + return interpreterSettingsRef; + } + + private URL[] recursiveBuildLibList(File path) throws MalformedURLException { + URL[] urls = new URL[0]; + if (path == null || !path.exists()) { + return urls; + } else if (path.getName().startsWith(".")) { + return urls; + } else if (path.isDirectory()) { + File[] files = path.listFiles(); + if (files != null) { + for (File f : files) { + urls = (URL[]) ArrayUtils.addAll(urls, recursiveBuildLibList(f)); + } + } + return urls; + } else { + return new URL[]{path.toURI().toURL()}; + } + } + + public List getRepositories() { + return this.interpreterRepositories; + } + + public void addRepository(String id, String url, boolean snapshot, Authentication auth, + Proxy proxy) throws IOException { + dependencyResolver.addRepo(id, url, snapshot, auth, proxy); + saveToFile(); + } + + public void removeRepository(String id) throws IOException { + dependencyResolver.delRepo(id); + saveToFile(); + } + + public void removeNoteInterpreterSettingBinding(String user, String noteId) { + synchronized (interpreterSettings) { + List settingIds = (interpreterBindings.containsKey(noteId) ? + interpreterBindings.remove(noteId) : + Collections.emptyList()); + for (String settingId : settingIds) { + this.removeInterpretersForNote(get(settingId), user, noteId); + } + } + } + + /** + * Change interpreter property and restart + */ + public void setPropertyAndRestart(String id, InterpreterOption option, Properties properties, + List dependencies) throws IOException { + synchronized (interpreterSettings) { + InterpreterSetting intpSetting = interpreterSettings.get(id); + if (intpSetting != null) { + try { + stopJobAllInterpreter(intpSetting); + + intpSetting.closeAndRemoveAllInterpreterGroups(); + intpSetting.setOption(option); + intpSetting.setProperties(properties); + intpSetting.setDependencies(dependencies); + loadInterpreterDependencies(intpSetting); + + saveToFile(); + } catch (Exception e) { + throw e; + } finally { + loadFromFile(); + } + } else { + throw new InterpreterException("Interpreter setting id " + id + " not found"); + } + } + } + + public void restart(String settingId, String noteId, String user) { + InterpreterSetting intpSetting = interpreterSettings.get(settingId); + Preconditions.checkNotNull(intpSetting); + + // restart interpreter setting in note page + if (noteIdIsExist(noteId) && intpSetting.getOption().isProcess()) { + intpSetting.closeAndRemoveInterpreterGroupByNoteId(noteId); + return; + } else { + // restart interpreter setting in interpreter setting page + restart(settingId, user); + } + + } + + private boolean noteIdIsExist(String noteId) { + return noteId == null ? false : true; + } + + public void restart(String id, String user) { + synchronized (interpreterSettings) { + InterpreterSetting intpSetting = interpreterSettings.get(id); + // Check if dependency in specified path is changed + // If it did, overwrite old dependency jar with new one + if (intpSetting != null) { + //clean up metaInfos + intpSetting.setInfos(null); + copyDependenciesFromLocalPath(intpSetting); + + stopJobAllInterpreter(intpSetting); + if (user.equals("anonymous")) { + intpSetting.closeAndRemoveAllInterpreterGroups(); + } else { + intpSetting.closeAndRemoveInterpreterGroupByUser(user); + } + + } else { + throw new InterpreterException("Interpreter setting id " + id + " not found"); + } + } + } + + public void restart(String id) { + restart(id, "anonymous"); + } + + private void stopJobAllInterpreter(InterpreterSetting intpSetting) { + if (intpSetting != null) { + for (InterpreterGroup intpGroup : intpSetting.getAllInterpreterGroups()) { + for (List interpreters : intpGroup.values()) { + for (Interpreter intp : interpreters) { + for (Job job : intp.getScheduler().getJobsRunning()) { + job.abort(); + job.setStatus(Status.ABORT); + logger.info("Job " + job.getJobName() + " aborted "); + } + for (Job job : intp.getScheduler().getJobsWaiting()) { + job.abort(); + job.setStatus(Status.ABORT); + logger.info("Job " + job.getJobName() + " aborted "); + } + } + } + } + } + } + + public InterpreterSetting get(String name) { + synchronized (interpreterSettings) { + return interpreterSettings.get(name); + } + } + + public void remove(String id) throws IOException { + synchronized (interpreterSettings) { + if (interpreterSettings.containsKey(id)) { + InterpreterSetting intp = interpreterSettings.get(id); + intp.closeAndRemoveAllInterpreterGroups(); + + interpreterSettings.remove(id); + for (List settings : interpreterBindings.values()) { + Iterator it = settings.iterator(); + while (it.hasNext()) { + String settingId = it.next(); + if (settingId.equals(id)) { + it.remove(); + } + } + } + saveToFile(); + } + } + + File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + id); + FileUtils.deleteDirectory(localRepoDir); + } + + /** + * Get interpreter settings + */ + public List get() { + synchronized (interpreterSettings) { + List orderedSettings = new LinkedList<>(); + + Map> nameInterpreterSettingMap = new HashMap<>(); + for (InterpreterSetting interpreterSetting : interpreterSettings.values()) { + String group = interpreterSetting.getGroup(); + if (!nameInterpreterSettingMap.containsKey(group)) { + nameInterpreterSettingMap.put(group, new ArrayList()); + } + nameInterpreterSettingMap.get(group).add(interpreterSetting); + } + + for (String groupName : interpreterGroupOrderList) { + List interpreterSettingList = + nameInterpreterSettingMap.remove(groupName); + if (null != interpreterSettingList) { + for (InterpreterSetting interpreterSetting : interpreterSettingList) { + orderedSettings.add(interpreterSetting); + } + } + } + + List settings = new ArrayList<>(); + + for (List interpreterSettingList : nameInterpreterSettingMap.values()) { + for (InterpreterSetting interpreterSetting : interpreterSettingList) { + settings.add(interpreterSetting); + } + } + + Collections.sort(settings, new Comparator() { + @Override + public int compare(InterpreterSetting o1, InterpreterSetting o2) { + return o1.getName().compareTo(o2.getName()); + } + }); + + orderedSettings.addAll(settings); + + return orderedSettings; + } + } + + public void close() { + List closeThreads = new LinkedList<>(); + synchronized (interpreterSettings) { + Collection intpSettings = interpreterSettings.values(); + for (final InterpreterSetting intpSetting : intpSettings) { + Thread t = new Thread() { + public void run() { + intpSetting.closeAndRemoveAllInterpreterGroups(); + } + }; + t.start(); + closeThreads.add(t); + } + } + + for (Thread t : closeThreads) { + try { + t.join(); + } catch (InterruptedException e) { + logger.error("Can't close interpreterGroup", e); + } + } + } + + public void shutdown() { + List closeThreads = new LinkedList<>(); + synchronized (interpreterSettings) { + Collection intpSettings = interpreterSettings.values(); + for (final InterpreterSetting intpSetting : intpSettings) { + Thread t = new Thread() { + public void run() { + intpSetting.shutdownAndRemoveAllInterpreterGroups(); + } + }; + t.start(); + closeThreads.add(t); + } + } + + for (Thread t : closeThreads) { + try { + t.join(); + } catch (InterruptedException e) { + logger.error("Can't close interpreterGroup", e); + } + } + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 224dd4b7697..c4b194bdfe3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -77,6 +77,7 @@ public class Note implements Serializable, ParagraphJobListener { private Map> angularObjects = new HashMap<>(); private transient InterpreterFactory factory; + private transient InterpreterSettingManager interpreterSettingManager; private transient JobListenerFactory jobListenerFactory; private transient NotebookRepo repo; private transient SearchService index; @@ -101,10 +102,12 @@ public class Note implements Serializable, ParagraphJobListener { public Note() { } - public Note(NotebookRepo repo, InterpreterFactory factory, JobListenerFactory jlFactory, + public Note(NotebookRepo repo, InterpreterFactory factory, + InterpreterSettingManager interpreterSettingManager, JobListenerFactory jlFactory, SearchService noteIndex, Credentials credentials, NoteEventListener noteEventListener) { this.repo = repo; this.factory = factory; + this.interpreterSettingManager = interpreterSettingManager; this.jobListenerFactory = jlFactory; this.index = noteIndex; this.noteEventListener = noteEventListener; @@ -117,7 +120,7 @@ private void generateId() { } private String getDefaultInterpreterName() { - InterpreterSetting setting = factory.getDefaultInterpreterSetting(getId()); + InterpreterSetting setting = interpreterSettingManager.getDefaultInterpreterSetting(getId()); return null != setting ? setting.getName() : StringUtils.EMPTY; } @@ -220,6 +223,15 @@ void setInterpreterFactory(InterpreterFactory factory) { } } + void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) { + this.interpreterSettingManager = interpreterSettingManager; + synchronized (paragraphs) { + for (Paragraph p : paragraphs) { + p.setInterpreterSettingManager(interpreterSettingManager); + } + } + } + public void initializeJobListenerForParagraph(Paragraph paragraph) { final Note paragraphNote = paragraph.getNote(); if (!paragraphNote.getId().equals(this.getId())) { @@ -272,7 +284,7 @@ Map> getAngularObjects() { * Add paragraph last. */ public Paragraph addParagraph(AuthenticationInfo authenticationInfo) { - Paragraph p = new Paragraph(this, this, factory); + Paragraph p = new Paragraph(this, this, factory, interpreterSettingManager); p.setAuthenticationInfo(authenticationInfo); setParagraphMagic(p, paragraphs.size()); synchronized (paragraphs) { @@ -292,7 +304,8 @@ public Paragraph addParagraph(AuthenticationInfo authenticationInfo) { void addCloneParagraph(Paragraph srcParagraph) { // Keep paragraph original ID - final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory); + final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory, + interpreterSettingManager); Map config = new HashMap<>(srcParagraph.getConfig()); Map param = new HashMap<>(srcParagraph.settings.getParams()); @@ -329,7 +342,7 @@ void addCloneParagraph(Paragraph srcParagraph) { * @param index index of paragraphs */ public Paragraph insertParagraph(int index, AuthenticationInfo authenticationInfo) { - Paragraph p = new Paragraph(this, this, factory); + Paragraph p = new Paragraph(this, this, factory, interpreterSettingManager); p.setAuthenticationInfo(authenticationInfo); setParagraphMagic(p, index); synchronized (paragraphs) { @@ -623,7 +636,7 @@ public List getParagraphs() { private void snapshotAngularObjectRegistry(String user) { angularObjects = new HashMap<>(); - List settings = factory.getInterpreterSettings(getId()); + List settings = interpreterSettingManager.getInterpreterSettings(getId()); if (settings == null || settings.size() == 0) { return; } @@ -638,7 +651,7 @@ private void snapshotAngularObjectRegistry(String user) { private void removeAllAngularObjectInParagraph(String user, String paragraphId) { angularObjects = new HashMap<>(); - List settings = factory.getInterpreterSettings(getId()); + List settings = interpreterSettingManager.getInterpreterSettings(getId()); if (settings == null || settings.size() == 0) { return; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index b853e070431..dfef6f45bb0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -77,6 +77,7 @@ public class Notebook implements NoteEventListener { private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; + private InterpreterSettingManager interpreterSettingManager; /** * Keep the order. */ @@ -102,13 +103,14 @@ public class Notebook implements NoteEventListener { */ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, SchedulerFactory schedulerFactory, InterpreterFactory replFactory, - JobListenerFactory jobListenerFactory, SearchService noteSearchService, - NotebookAuthorization notebookAuthorization, Credentials credentials) - throws IOException, SchedulerException { + InterpreterSettingManager interpreterSettingManager, JobListenerFactory jobListenerFactory, + SearchService noteSearchService, NotebookAuthorization notebookAuthorization, + Credentials credentials) throws IOException, SchedulerException { this.conf = conf; this.notebookRepo = notebookRepo; this.schedulerFactory = schedulerFactory; this.replFactory = replFactory; + this.interpreterSettingManager = interpreterSettingManager; this.jobListenerFactory = jobListenerFactory; this.noteSearchService = noteSearchService; this.notebookAuthorization = notebookAuthorization; @@ -138,7 +140,7 @@ public Note createNote(AuthenticationInfo subject) throws IOException { Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null"); Note note; if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) { - note = createNote(replFactory.getDefaultInterpreterSettingList(), subject); + note = createNote(interpreterSettingManager.getDefaultInterpreterSettingList(), subject); } else { note = createNote(null, subject); } @@ -154,7 +156,7 @@ public Note createNote(AuthenticationInfo subject) throws IOException { public Note createNote(List interpreterIds, AuthenticationInfo subject) throws IOException { Note note = - new Note(notebookRepo, replFactory, jobListenerFactory, + new Note(notebookRepo, replFactory, interpreterSettingManager, jobListenerFactory, noteSearchService, credentials, this); note.setNoteNameListener(folders); @@ -270,14 +272,15 @@ public void bindInterpretersToNote(String user, String id, List interpre throws IOException { Note note = getNote(id); if (note != null) { - List currentBindings = replFactory.getInterpreterSettings(id); + List currentBindings = + interpreterSettingManager.getInterpreterSettings(id); for (InterpreterSetting setting : currentBindings) { if (!interpreterSettingIds.contains(setting.getId())) { fireUnbindInterpreter(note, setting); } } - replFactory.setInterpreters(user, note.getId(), interpreterSettingIds); + interpreterSettingManager.setInterpreters(user, note.getId(), interpreterSettingIds); // comment out while note.getNoteReplLoader().setInterpreters(...) do the same // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); } @@ -286,7 +289,7 @@ public void bindInterpretersToNote(String user, String id, List interpre List getBindedInterpreterSettingsIds(String id) { Note note = getNote(id); if (note != null) { - return getInterpreterFactory().getInterpreters(note.getId()); + return interpreterSettingManager.getInterpreters(note.getId()); } else { return new LinkedList<>(); } @@ -295,7 +298,7 @@ List getBindedInterpreterSettingsIds(String id) { public List getBindedInterpreterSettings(String id) { Note note = getNote(id); if (note != null) { - return replFactory.getInterpreterSettings(note.getId()); + return interpreterSettingManager.getInterpreterSettings(note.getId()); } else { return new LinkedList<>(); } @@ -328,12 +331,12 @@ public void removeNote(String id, AuthenticationInfo subject) { note = notes.remove(id); folders.removeNote(note); } - replFactory.removeNoteInterpreterSettingBinding(subject.getUser(), id); + interpreterSettingManager.removeNoteInterpreterSettingBinding(subject.getUser(), id); noteSearchService.deleteIndexDocs(note); notebookAuthorization.removeNote(id); // remove from all interpreter instance's angular object registry - for (InterpreterSetting settings : replFactory.get()) { + for (InterpreterSetting settings : interpreterSettingManager.get()) { AngularObjectRegistry registry = settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry(); if (registry instanceof RemoteAngularObjectRegistry) { @@ -467,6 +470,7 @@ public Note loadNoteFromRepo(String id, AuthenticationInfo subject) { note.setCredentials(this.credentials); note.setInterpreterFactory(replFactory); + note.setInterpreterSettingManager(interpreterSettingManager); note.setJobListenerFactory(jobListenerFactory); note.setNotebookRepo(notebookRepo); @@ -510,7 +514,7 @@ public Note loadNoteFromRepo(String id, AuthenticationInfo subject) { for (String name : angularObjectSnapshot.keySet()) { SnapshotAngularObject snapshot = angularObjectSnapshot.get(name); - List settings = replFactory.get(); + List settings = interpreterSettingManager.get(); for (InterpreterSetting setting : settings) { InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId()); if (intpGroup.getId().equals(snapshot.getIntpGroupId())) { @@ -754,9 +758,10 @@ public List> getJobListByNoteId(String noteId) { // set interpreter bind type String interpreterGroupName = null; - if (replFactory.getInterpreterSettings(jobNote.getId()) != null - && replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) { - interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName(); + if (interpreterSettingManager.getInterpreterSettings(jobNote.getId()) != null + && interpreterSettingManager.getInterpreterSettings(jobNote.getId()).size() >= 1) { + interpreterGroupName = + interpreterSettingManager.getInterpreterSettings(jobNote.getId()).get(0).getName(); } // note json object root information. @@ -830,9 +835,10 @@ public List> getJobListByUnixTime(boolean needsReload, // set interpreter bind type String interpreterGroupName = null; - if (replFactory.getInterpreterSettings(note.getId()) != null - && replFactory.getInterpreterSettings(note.getId()).size() >= 1) { - interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName(); + if (interpreterSettingManager.getInterpreterSettings(note.getId()) != null + && interpreterSettingManager.getInterpreterSettings(note.getId()).size() >= 1) { + interpreterGroupName = + interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getName(); } // not update and not running -> pass @@ -882,9 +888,9 @@ public void execute(JobExecutionContext context) throws JobExecutionException { logger.error(e.getMessage(), e); } if (releaseResource) { - for (InterpreterSetting setting : notebook.getInterpreterFactory() + for (InterpreterSetting setting : notebook.getInterpreterSettingManager() .getInterpreterSettings(note.getId())) { - notebook.getInterpreterFactory().restart(setting.getId()); + notebook.getInterpreterSettingManager().restart(setting.getId()); } } } @@ -949,6 +955,10 @@ public InterpreterFactory getInterpreterFactory() { return replFactory; } + public InterpreterSettingManager getInterpreterSettingManager() { + return interpreterSettingManager; + } + public NotebookAuthorization getNotebookAuthorization() { return notebookAuthorization; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 12501946730..92a1d26f6c4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -53,6 +53,7 @@ public class Paragraph extends Job implements Serializable, Cloneable { private static Logger logger = LoggerFactory.getLogger(Paragraph.class); private transient InterpreterFactory factory; + private transient InterpreterSettingManager interpreterSettingManager; private transient Note note; private transient AuthenticationInfo authenticationInfo; private transient Map userParagraphMap = Maps.newHashMap(); // personalized @@ -85,10 +86,11 @@ public class Paragraph extends Job implements Serializable, Cloneable { } public Paragraph(String paragraphId, Note note, JobListener listener, - InterpreterFactory factory) { + InterpreterFactory factory, InterpreterSettingManager interpreterSettingManager) { super(paragraphId, generateId(), listener); this.note = note; this.factory = factory; + this.interpreterSettingManager = interpreterSettingManager; title = null; text = null; authenticationInfo = null; @@ -98,10 +100,12 @@ public Paragraph(String paragraphId, Note note, JobListener listener, config = new HashMap<>(); } - public Paragraph(Note note, JobListener listener, InterpreterFactory factory) { + public Paragraph(Note note, JobListener listener, InterpreterFactory factory, + InterpreterSettingManager interpreterSettingManager) { super(generateId(), listener); this.note = note; this.factory = factory; + this.interpreterSettingManager = interpreterSettingManager; title = null; text = null; authenticationInfo = null; @@ -248,7 +252,7 @@ public Interpreter getCurrentRepl() { public List getInterpreterCompletion() { List completion = new LinkedList(); - for (InterpreterSetting intp : factory.getInterpreterSettings(note.getId())) { + for (InterpreterSetting intp : interpreterSettingManager.getInterpreterSettings(note.getId())) { List intInfo = intp.getInterpreterInfos(); if (intInfo.size() > 1) { for (InterpreterInfo info : intInfo) { @@ -291,6 +295,10 @@ public void setInterpreterFactory(InterpreterFactory factory) { this.factory = factory; } + public void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) { + this.interpreterSettingManager = interpreterSettingManager; + } + public InterpreterResult getResult() { return (InterpreterResult) getReturn(); } @@ -415,7 +423,7 @@ private boolean noteHasUser() { } private boolean noteHasInterpreters() { - return !factory.getInterpreterSettings(note.getId()).isEmpty(); + return !interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty(); } private boolean interpreterHasUser(InterpreterSetting intp) { @@ -429,7 +437,7 @@ private boolean isUserAuthorizedToAccessInterpreter(InterpreterOption intpOpt) { private InterpreterSetting getInterpreterSettingById(String id) { InterpreterSetting setting = null; - for (InterpreterSetting i : factory.getInterpreterSettings(note.getId())) { + for (InterpreterSetting i : interpreterSettingManager.getInterpreterSettings(note.getId())) { if (id.startsWith(i.getId())) { setting = i; break; @@ -503,8 +511,9 @@ private InterpreterContext getInterpreterContextWithoutRunner(InterpreterOutput AngularObjectRegistry registry = null; ResourcePool resourcePool = null; - if (!factory.getInterpreterSettings(note.getId()).isEmpty()) { - InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0); + if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) { + InterpreterSetting intpGroup = + interpreterSettingManager.getInterpreterSettings(note.getId()).get(0); registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry(); resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool(); } @@ -531,8 +540,9 @@ private InterpreterContext getInterpreterContext(InterpreterOutput output) { AngularObjectRegistry registry = null; ResourcePool resourcePool = null; - if (!factory.getInterpreterSettings(note.getId()).isEmpty()) { - InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0); + if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) { + InterpreterSetting intpGroup = + interpreterSettingManager.getInterpreterSettings(note.getId()).get(0); registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry(); resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool(); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 99cdecaeabe..c22001964ff 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -49,6 +49,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { private SchedulerFactory schedulerFactory; private DependencyResolver depResolver; private InterpreterFactory factory; + private InterpreterSettingManager interpreterSettingManager; private VFSNotebookRepo notebookRepo; private Notebook notebook; private HeliumApplicationFactory heliumAppFactory; @@ -84,8 +85,8 @@ public void setUp() throws Exception { heliumAppFactory = new HeliumApplicationFactory(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, - new InterpreterOption(true), null, null, heliumAppFactory, depResolver, false); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, heliumAppFactory, depResolver, false, interpreterSettingManager); HashMap env = new HashMap<>(); env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); factory.setEnv(env); @@ -98,6 +99,7 @@ public void setUp() throws Exception { notebookRepo, schedulerFactory, factory, + interpreterSettingManager, this, search, notebookAuthorization, @@ -112,7 +114,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - List settings = factory.get(); + List settings = interpreterSettingManager.get(); for (InterpreterSetting setting : settings) { for (InterpreterGroup intpGroup : setting.getAllInterpreterGroups()) { intpGroup.close(); @@ -138,7 +140,7 @@ public void testLoadRunUnloadApplication() "", ""); Note note1 = notebook.createNote(anonymous); - factory.setInterpreters("user", note1.getId(),factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", note1.getId(),interpreterSettingManager.getDefaultInterpreterSettingList()); Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS); @@ -184,7 +186,7 @@ public void testUnloadOnParagraphRemove() throws IOException { "", ""); Note note1 = notebook.createNote(anonymous); - factory.setInterpreters("user", note1.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS); @@ -224,7 +226,7 @@ public void testUnloadOnInterpreterUnbind() throws IOException { "", ""); Note note1 = notebook.createNote(anonymous); - notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList()); + notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS); @@ -285,7 +287,7 @@ public void testUnloadOnInterpreterRestart() throws IOException { "", ""); Note note1 = notebook.createNote(anonymous); - notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList()); + notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); String mock1IntpSettingId = null; for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) { if (setting.getName().equals("mock1")) { @@ -312,7 +314,7 @@ public void testUnloadOnInterpreterRestart() throws IOException { Thread.yield(); } // when restart interpreter - factory.restart(mock1IntpSettingId); + interpreterSettingManager.restart(mock1IntpSettingId); while (app.getStatus() == ApplicationState.Status.LOADED) { Thread.yield(); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 7522366b474..a8da7d39ee8 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -68,6 +68,7 @@ public class InterpreterFactoryTest { private InterpreterFactory factory; + private InterpreterSettingManager interpreterSettingManager; private File tmpDir; private ZeppelinConfiguration conf; private InterpreterContext context; @@ -102,13 +103,14 @@ public void setUp() throws Exception { conf = new ZeppelinConfiguration(); schedulerFactory = new SchedulerFactory(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); context = new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); notebookAuthorization = NotebookAuthorization.init(conf); - notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, jobListenerFactory, search, + notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, jobListenerFactory, search, notebookAuthorization, null); } @@ -119,7 +121,7 @@ public void tearDown() throws Exception { @Test public void testBasic() { - List all = factory.get(); + List all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { if (setting.getName().equals("mock1")) { @@ -137,17 +139,18 @@ public void testBasic() { assertNotNull("get Interpreter", interpreterGroup.get("session").get(0)); // try to get unavailable interpreter - assertNull(factory.get("unknown")); + assertNull(interpreterSettingManager.get("unknown")); // restart interpreter - factory.restart(mock1Setting.getId()); + interpreterSettingManager.restart(mock1Setting.getId()); assertNull(mock1Setting.getInterpreterGroup("user", "sharedProcess").get("session")); } @Test public void testRemoteRepl() throws Exception { - factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false); - List all = factory.get(); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + List all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { if (setting.getName().equals("mock1")) { @@ -174,8 +177,9 @@ public void testRemoteRepl() throws Exception { */ @Test public void testRestartInterpreterInScopedMode() throws Exception { - factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false); - List all = factory.get(); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + List all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { if (setting.getName().equals("mock1")) { @@ -210,8 +214,9 @@ public void testRestartInterpreterInScopedMode() throws Exception { */ @Test public void testRestartInterpreterInIsolatedMode() throws Exception { - factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false); - List all = factory.get(); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + List all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { if (setting.getName().equals("mock1")) { @@ -243,21 +248,21 @@ public void testRestartInterpreterInIsolatedMode() throws Exception { @Test public void testFactoryDefaultList() throws IOException, RepositoryException { // get default settings - List all = factory.getDefaultInterpreterSettingList(); - assertTrue(factory.get().size() >= all.size()); + List all = interpreterSettingManager.getDefaultInterpreterSettingList(); + assertTrue(interpreterSettingManager.get().size() >= all.size()); } @Test public void testExceptions() throws InterpreterException, IOException, RepositoryException { - List all = factory.getDefaultInterpreterSettingList(); + List all = interpreterSettingManager.getDefaultInterpreterSettingList(); // add setting with null option & properties expected nullArgumentException.class try { - factory.add("mock2", new ArrayList(), new LinkedList(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null); + interpreterSettingManager.add("mock2", new ArrayList(), new LinkedList(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null); } catch(NullArgumentException e) { assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage()); } try { - factory.add("mock2", new ArrayList(), new LinkedList(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null); + interpreterSettingManager.add("mock2", new ArrayList(), new LinkedList(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null); } catch (NullArgumentException e){ assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage()); } @@ -267,22 +272,23 @@ public void testExceptions() throws InterpreterException, IOException, Repositor @Test public void testSaveLoad() throws IOException, RepositoryException { // interpreter settings - int numInterpreters = factory.get().size(); + int numInterpreters = interpreterSettingManager.get().size(); // check if file saved assertTrue(new File(conf.getInterpreterSettingPath()).exists()); - factory.createNewSetting("new-mock1", "mock1", new LinkedList(), new InterpreterOption(false), new Properties()); - assertEquals(numInterpreters + 1, factory.get().size()); + interpreterSettingManager.createNewSetting("new-mock1", "mock1", new LinkedList(), new InterpreterOption(false), new Properties()); + assertEquals(numInterpreters + 1, interpreterSettingManager.get().size()); - InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver, false); - assertEquals(numInterpreters + 1, factory2.get().size()); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + + assertEquals(numInterpreters + 1, interpreterSettingManager.get().size()); } @Test public void testInterpreterSettingPropertyClass() throws IOException, RepositoryException { // check if default interpreter reference's property type is map - Map interpreterSettingRefs = factory.getAvailableInterpreterSettings(); + Map interpreterSettingRefs = interpreterSettingManager.getAvailableInterpreterSettings(); InterpreterSetting intpSetting = interpreterSettingRefs.get("mock1"); Map intpProperties = (Map) intpSetting.getProperties(); @@ -293,7 +299,7 @@ public void testInterpreterSettingPropertyClass() throws IOException, Repository properties.put("key1", "value1"); properties.put("key2", "value2"); - factory.createNewSetting("newMock", "mock1", new LinkedList(), new InterpreterOption(false), properties); + interpreterSettingManager.createNewSetting("newMock", "mock1", new LinkedList(), new InterpreterOption(false), properties); String confFilePath = conf.getInterpreterSettingPath(); byte[] encoded = Files.readAllBytes(Paths.get(confFilePath)); @@ -312,20 +318,21 @@ public void testInterpreterSettingPropertyClass() throws IOException, Repository @Test public void testInterpreterAliases() throws IOException, RepositoryException { - factory = new InterpreterFactory(conf, null, null, null, depResolver, false); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null); final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null); - factory.add("group1", new ArrayList() {{ + interpreterSettingManager.add("group1", new ArrayList() {{ add(info1); }}, new ArrayList(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null); - factory.add("group2", new ArrayList(){{ + interpreterSettingManager.add("group2", new ArrayList(){{ add(info2); }}, new ArrayList(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path2", null); - final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList(), new InterpreterOption(true), new Properties()); - final InterpreterSetting setting2 = factory.createNewSetting("test-group2", "group1", new ArrayList(), new InterpreterOption(true), new Properties()); + final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList(), new InterpreterOption(true), new Properties()); + final InterpreterSetting setting2 = interpreterSettingManager.createNewSetting("test-group2", "group1", new ArrayList(), new InterpreterOption(true), new Properties()); - factory.setInterpreters("user", "note", new ArrayList() {{ + interpreterSettingManager.setInterpreters("user", "note", new ArrayList() {{ add(setting1.getId()); add(setting2.getId()); }}); @@ -336,20 +343,21 @@ public void testInterpreterAliases() throws IOException, RepositoryException { @Test public void testMultiUser() throws IOException, RepositoryException { - factory = new InterpreterFactory(conf, null, null, null, depResolver, true); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, true, interpreterSettingManager); final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null); - factory.add("group1", new ArrayList(){{ + interpreterSettingManager.add("group1", new ArrayList(){{ add(info1); }}, new ArrayList(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null); InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED); - final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList(), perUserInterpreterOption, new Properties()); + final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList(), perUserInterpreterOption, new Properties()); - factory.setInterpreters("user1", "note", new ArrayList() {{ + interpreterSettingManager.setInterpreters("user1", "note", new ArrayList() {{ add(setting1.getId()); }}); - factory.setInterpreters("user2", "note", new ArrayList() {{ + interpreterSettingManager.setInterpreters("user2", "note", new ArrayList() {{ add(setting1.getId()); }}); @@ -360,7 +368,7 @@ public void testMultiUser() throws IOException, RepositoryException { @Test public void testInvalidInterpreterSettingName() { try { - factory.createNewSetting("new.mock1", "mock1", new LinkedList(), new InterpreterOption(false), new Properties()); + interpreterSettingManager.createNewSetting("new.mock1", "mock1", new LinkedList(), new InterpreterOption(false), new Properties()); fail("expect fail because of invalid InterpreterSetting Name"); } catch (IOException e) { assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage()); @@ -371,39 +379,40 @@ public void testInvalidInterpreterSettingName() { @Test public void getEditorSetting() throws IOException, RepositoryException, SchedulerException { List intpIds = new ArrayList<>(); - for(InterpreterSetting intpSetting: factory.get()) { + for(InterpreterSetting intpSetting: interpreterSettingManager.get()) { if (intpSetting.getName().startsWith("mock1")) { intpIds.add(intpSetting.getId()); } } Note note = notebook.createNote(intpIds, new AuthenticationInfo("anonymous")); + Interpreter interpreter = factory.getInterpreter("user1", note.getId(), "mock11"); // get editor setting from interpreter-setting.json - Map editor = factory.getEditorSetting("user1", note.getId(), "mock11"); + Map editor = interpreterSettingManager.getEditorSetting(interpreter, "user1", note.getId(), "mock11"); assertEquals("java", editor.get("language")); // when interpreter is not loaded via interpreter-setting.json // or editor setting doesn't exit - editor = factory.getEditorSetting("user1", note.getId(), "mock1"); + editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock1"),"user1", note.getId(), "mock1"); assertEquals(null, editor.get("language")); // when interpreter is not bound to note - editor = factory.getEditorSetting("user1", note.getId(), "mock2"); + editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock11"),"user1", note.getId(), "mock2"); assertEquals("text", editor.get("language")); } @Test public void registerCustomInterpreterRunner() throws IOException { - InterpreterFactory spyFactory = spy(factory); + InterpreterSettingManager spyInterpreterSettingManager = spy(interpreterSettingManager); - doNothing().when(spyFactory).saveToFile(); + doNothing().when(spyInterpreterSettingManager).saveToFile(); ArrayList interpreterInfos1 = new ArrayList<>(); interpreterInfos1.add(new InterpreterInfo("name1.class", "name1", true, Maps.newHashMap())); - spyFactory.add("normalGroup1", interpreterInfos1, Lists.newArrayList(), new InterpreterOption(true), Maps.newHashMap(), "/normalGroup1", null); + spyInterpreterSettingManager.add("normalGroup1", interpreterInfos1, Lists.newArrayList(), new InterpreterOption(true), Maps.newHashMap(), "/normalGroup1", null); - spyFactory.createNewSetting("normalGroup1", "normalGroup1", Lists.newArrayList(), new InterpreterOption(true), new Properties()); + spyInterpreterSettingManager.createNewSetting("normalGroup1", "normalGroup1", Lists.newArrayList(), new InterpreterOption(true), new Properties()); ArrayList interpreterInfos2 = new ArrayList<>(); interpreterInfos2.add(new InterpreterInfo("name1.class", "name1", true, Maps.newHashMap())); @@ -412,13 +421,13 @@ public void registerCustomInterpreterRunner() throws IOException { when(mockInterpreterRunner.getPath()).thenReturn("custom-linux-path.sh"); - spyFactory.add("customGroup1", interpreterInfos2, Lists.newArrayList(), new InterpreterOption(true), Maps.newHashMap(), "/customGroup1", mockInterpreterRunner); + spyInterpreterSettingManager.add("customGroup1", interpreterInfos2, Lists.newArrayList(), new InterpreterOption(true), Maps.newHashMap(), "/customGroup1", mockInterpreterRunner); - spyFactory.createNewSetting("customGroup1", "customGroup1", Lists.newArrayList(), new InterpreterOption(true), new Properties()); + spyInterpreterSettingManager.createNewSetting("customGroup1", "customGroup1", Lists.newArrayList(), new InterpreterOption(true), new Properties()); - spyFactory.setInterpreters("anonymous", "noteCustome", spyFactory.getDefaultInterpreterSettingList()); + spyInterpreterSettingManager.setInterpreters("anonymous", "noteCustome", spyInterpreterSettingManager.getDefaultInterpreterSettingList()); - spyFactory.getInterpreter("anonymous", "noteCustome", "customGroup1"); + factory.getInterpreter("anonymous", "noteCustome", "customGroup1"); verify(mockInterpreterRunner, times(1)).getPath(); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java index 8325e8ae0ab..27aa63373d7 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java @@ -19,6 +19,7 @@ import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.search.SearchService; @@ -59,6 +60,9 @@ public class FolderTest { @Mock InterpreterFactory interpreterFactory; + @Mock + InterpreterSettingManager interpreterSettingManager; + Folder folder; Note note1; @@ -67,13 +71,13 @@ public class FolderTest { @Before public void createFolderAndNotes() { - note1 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + note1 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); note1.setName("this/is/a/folder/note1"); - note2 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + note2 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); note2.setName("this/is/a/folder/note2"); - note3 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + note3 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); note3.setName("this/is/a/folder/note3"); folder = new Folder("this/is/a/folder"); @@ -114,7 +118,7 @@ public void folderIdTest() { @Test public void addNoteTest() { - Note note4 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note4 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); note4.setName("this/is/a/folder/note4"); folder.addNote(note4); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderViewTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderViewTest.java index 3ed14b6b9e5..cdba75d019e 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderViewTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderViewTest.java @@ -19,6 +19,7 @@ import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.search.SearchService; @@ -61,6 +62,9 @@ public class FolderViewTest { @Mock InterpreterFactory interpreterFactory; + @Mock + InterpreterSettingManager interpreterSettingManager; + FolderView folderView; Note note1; @@ -85,7 +89,7 @@ public class FolderViewTest { Note abNote2; private Note createNote() { - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); note.setNoteNameListener(folderView); return note; } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index 320a5b56424..fc8c8bc66a8 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -29,6 +29,7 @@ import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; import org.apache.zeppelin.interpreter.mock.MockInterpreter11; @@ -47,6 +48,7 @@ public class NoteInterpreterLoaderTest { private File tmpDir; private ZeppelinConfiguration conf; private InterpreterFactory factory; + private InterpreterSettingManager interpreterSettingManager; private DependencyResolver depResolver; @Before @@ -66,7 +68,8 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "group2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); } @After @@ -77,7 +80,7 @@ public void tearDown() throws Exception { @Test public void testGetInterpreter() throws IOException { - factory.setInterpreters("user", "note", factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", "note", interpreterSettingManager.getDefaultInterpreterSettingList()); // when there're no interpreter selection directive assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", null).getClassName()); @@ -96,20 +99,20 @@ public void testGetInterpreter() throws IOException { assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "group1.mock11").getClassName()); assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2.mock2").getClassName()); - factory.closeNote("user", "note"); + interpreterSettingManager.closeNote("user", "note"); } @Test public void testNoteSession() throws IOException { - factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList()); - factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.SCOPED); + interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.SCOPED); - factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList()); - factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.SCOPED); + interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.SCOPED); // interpreters are not created before accessing it - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); + assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); + assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); factory.getInterpreter("user", "noteA", null).open(); factory.getInterpreter("user", "noteB", null).open(); @@ -119,35 +122,35 @@ public void testNoteSession() throws IOException { factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId())); // interpreters are created after accessing it - assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); - assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); + assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA")); + assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB")); // invalid close - factory.closeNote("user", "note"); - assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA")); - assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB")); + interpreterSettingManager.closeNote("user", "note"); + assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA")); + assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB")); // when - factory.closeNote("user", "noteA"); - factory.closeNote("user", "noteB"); + interpreterSettingManager.closeNote("user", "noteA"); + interpreterSettingManager.closeNote("user", "noteB"); // interpreters are destroyed after close - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB")); + assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA")); + assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB")); } @Test public void testNotePerInterpreterProcess() throws IOException { - factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList()); - factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.ISOLATED); + interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.ISOLATED); - factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList()); - factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.ISOLATED); + interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.ISOLATED); // interpreters are not created before accessing it - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session")); + assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session")); + assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session")); factory.getInterpreter("user", "noteA", null).open(); factory.getInterpreter("user", "noteB", null).open(); @@ -158,29 +161,29 @@ public void testNotePerInterpreterProcess() throws IOException { factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId())); // interpreters are created after accessing it - assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session")); - assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session")); + assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session")); + assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session")); // when - factory.closeNote("user", "noteA"); - factory.closeNote("user", "noteB"); + interpreterSettingManager.closeNote("user", "noteA"); + interpreterSettingManager.closeNote("user", "noteB"); // interpreters are destroyed after close - assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session")); - assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session")); + assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session")); + assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session")); } @Test public void testNoteInterpreterCloseForAll() throws IOException { - factory.setInterpreters("user", "FitstNote", factory.getDefaultInterpreterSettingList()); - factory.getInterpreterSettings("FitstNote").get(0).getOption().setPerNote(InterpreterOption.SCOPED); + interpreterSettingManager.setInterpreters("user", "FitstNote", interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getOption().setPerNote(InterpreterOption.SCOPED); - factory.setInterpreters("user", "yourFirstNote", factory.getDefaultInterpreterSettingList()); - factory.getInterpreterSettings("yourFirstNote").get(0).getOption().setPerNote(InterpreterOption.ISOLATED); + interpreterSettingManager.setInterpreters("user", "yourFirstNote", interpreterSettingManager.getDefaultInterpreterSettingList()); + interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getOption().setPerNote(InterpreterOption.ISOLATED); // interpreters are not created before accessing it - assertNull(factory.getInterpreterSettings("FitstNote").get(0).getInterpreterGroup("user", "FitstNote").get("FitstNote")); - assertNull(factory.getInterpreterSettings("yourFirstNote").get(0).getInterpreterGroup("user", "yourFirstNote").get("yourFirstNote")); + assertNull(interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getInterpreterGroup("user", "FitstNote").get("FitstNote")); + assertNull(interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getInterpreterGroup("user", "yourFirstNote").get("yourFirstNote")); Interpreter firstNoteIntp = factory.getInterpreter("user", "FitstNote", "group1.mock1"); Interpreter yourFirstNoteIntp = factory.getInterpreter("user", "yourFirstNote", "group1.mock1"); @@ -191,7 +194,7 @@ public void testNoteInterpreterCloseForAll() throws IOException { assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen()); assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen()); - factory.closeNote("user", "FitstNote"); + interpreterSettingManager.closeNote("user", "FitstNote"); assertFalse(((LazyOpenInterpreter)firstNoteIntp).isOpen()); assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen()); @@ -203,13 +206,13 @@ public void testNoteInterpreterCloseForAll() throws IOException { assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen()); // invalid check - factory.closeNote("invalid", "Note"); + interpreterSettingManager.closeNote("invalid", "Note"); assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen()); assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen()); // invalid contains value check - factory.closeNote("u", "Note"); + interpreterSettingManager.closeNote("u", "Note"); assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen()); assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen()); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java index ad6031e9a1c..708acd6e315 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java @@ -20,6 +20,7 @@ import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.search.SearchService; @@ -60,6 +61,9 @@ public class NoteTest { @Mock InterpreterFactory interpreterFactory; + @Mock + InterpreterSettingManager interpreterSettingManager; + private AuthenticationInfo anonymous = new AuthenticationInfo("anonymous"); @Test @@ -68,7 +72,7 @@ public void runNormalTest() { when(interpreter.getScheduler()).thenReturn(scheduler); String pText = "%spark sc.version"; - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS); p.setText(pText); @@ -84,7 +88,7 @@ public void runNormalTest() { @Test public void addParagraphWithEmptyReplNameTest() { - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS); assertNull(p.getText()); @@ -94,7 +98,7 @@ public void addParagraphWithEmptyReplNameTest() { public void addParagraphWithLastReplNameTest() { when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter); - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS); p1.setText("%spark "); Paragraph p2 = note.addParagraph(AuthenticationInfo.ANONYMOUS); @@ -106,7 +110,7 @@ public void addParagraphWithLastReplNameTest() { public void insertParagraphWithLastReplNameTest() { when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter); - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS); p1.setText("%spark "); Paragraph p2 = note.insertParagraph(note.getParagraphs().size(), AuthenticationInfo.ANONYMOUS); @@ -118,7 +122,7 @@ public void insertParagraphWithLastReplNameTest() { public void insertParagraphWithInvalidReplNameTest() { when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("invalid"))).thenReturn(null); - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS); p1.setText("%invalid "); Paragraph p2 = note.insertParagraph(note.getParagraphs().size(), AuthenticationInfo.ANONYMOUS); @@ -131,7 +135,7 @@ public void clearAllParagraphOutputTest() { when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("md"))).thenReturn(interpreter); when(interpreter.getScheduler()).thenReturn(scheduler); - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS); InterpreterResult result = new InterpreterResult(InterpreterResult.Code.SUCCESS, InterpreterResult.Type.TEXT, "result"); p1.setResult(result); @@ -147,7 +151,7 @@ public void clearAllParagraphOutputTest() { @Test public void getFolderIdTest() { - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); // Ordinary case test note.setName("this/is/a/folder/noteName"); assertEquals("this/is/a/folder", note.getFolderId()); @@ -163,7 +167,7 @@ public void getFolderIdTest() { @Test public void getNameWithoutPathTest() { - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); // Notes in the root folder note.setName("noteOnRootFolder"); assertEquals("noteOnRootFolder", note.getNameWithoutPath()); @@ -178,7 +182,7 @@ public void getNameWithoutPathTest() { @Test public void isTrashTest() { - Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); // Notes in the root folder note.setName("noteOnRootFolder"); assertFalse(note.isTrash()); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 4bc94597f54..78903a9baa0 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -64,6 +64,7 @@ public class NotebookTest implements JobListenerFactory{ private Notebook notebook; private NotebookRepo notebookRepo; private InterpreterFactory factory; + private InterpreterSettingManager interpreterSettingManager; private DependencyResolver depResolver; private NotebookAuthorization notebookAuthorization; private Credentials credentials; @@ -91,14 +92,15 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(false)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); notebookAuthorization = NotebookAuthorization.init(conf); credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath()); - notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, this, search, + notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search, notebookAuthorization, credentials); } @@ -110,7 +112,7 @@ public void tearDown() throws Exception { @Test public void testSelectingReplImplementation() throws IOException { Note note = notebook.createNote(anonymous); - factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); // run with default repl Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS); @@ -236,7 +238,8 @@ public void testPersist() throws IOException, SchedulerException, RepositoryExce Notebook notebook2 = new Notebook( conf, notebookRepo, schedulerFactory, - new InterpreterFactory(conf, null, null, null, depResolver, false), this, null, null, null); + new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager), + interpreterSettingManager, null, null, null, null); assertEquals(1, notebook2.getAllNotes().size()); notebook.removeNote(note.getId(), anonymous); @@ -292,7 +295,7 @@ public void testRunBlankParagraph() throws IOException, SchedulerException, Inte @Test public void testRunAll() throws IOException { Note note = notebook.createNote(anonymous); - factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); // p1 Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS); @@ -331,7 +334,7 @@ public void testRunAll() throws IOException { public void testSchedule() throws InterruptedException, IOException { // create a note and a paragraph Note note = notebook.createNote(anonymous); - factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS); Map config = new HashMap<>(); @@ -364,7 +367,7 @@ public void testSchedule() throws InterruptedException, IOException { public void testSchedulePoolUsage() throws InterruptedException, IOException { // create a note and a paragraph Note note = notebook.createNote(anonymous); - factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS); Map config = Maps.newHashMap(); p.setConfig(config); @@ -432,7 +435,7 @@ private String getResultString(InterpreterResult result) { public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{ // create a note and a paragraph Note note = notebook.createNote(anonymous); - factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS); Map config = new HashMap<>(); @@ -485,7 +488,7 @@ public void testAutoRestartInterpreterAfterSchedule() throws InterruptedExceptio public void testExportAndImportNote() throws IOException, CloneNotSupportedException, InterruptedException, InterpreterException, SchedulerException, RepositoryException { Note note = notebook.createNote(anonymous); - factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); final Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS); String simpleText = "hello world"; @@ -524,7 +527,7 @@ public void testExportAndImportNote() throws IOException, CloneNotSupportedExcep public void testCloneNote() throws IOException, CloneNotSupportedException, InterruptedException, InterpreterException, SchedulerException, RepositoryException { Note note = notebook.createNote(anonymous); - factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); final Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS); p.setText("hello world"); @@ -558,7 +561,7 @@ public void testCloneNote() throws IOException, CloneNotSupportedException, public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedException, InterruptedException { Note note = notebook.createNote(anonymous); - factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); Note cloneNote = notebook.cloneNote(note.getId(), null, anonymous); assertEquals(cloneNote.getName(), "Note " + cloneNote.getId()); @@ -570,7 +573,7 @@ public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedExcep public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException, InterruptedException { Note note = notebook.createNote(anonymous); - factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); final Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS); p.setText("hello world"); @@ -595,7 +598,7 @@ public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSuppo @Test public void testResourceRemovealOnParagraphNoteRemove() throws IOException { Note note = notebook.createNote(anonymous); - factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) { intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId())); } @@ -624,9 +627,9 @@ public void testAngularObjectRemovalOnNotebookRemove() throws InterruptedExcepti IOException { // create a note and a paragraph Note note = notebook.createNote(anonymous); - factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); - AngularObjectRegistry registry = factory + AngularObjectRegistry registry = interpreterSettingManager .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); @@ -657,9 +660,9 @@ public void testAngularObjectRemovalOnParagraphRemove() throws InterruptedExcept IOException { // create a note and a paragraph Note note = notebook.createNote(anonymous); - factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); - AngularObjectRegistry registry = factory + AngularObjectRegistry registry = interpreterSettingManager .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); @@ -691,9 +694,9 @@ public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedExc IOException { // create a note and a paragraph Note note = notebook.createNote(anonymous); - factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); - AngularObjectRegistry registry = factory + AngularObjectRegistry registry = interpreterSettingManager .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); @@ -703,8 +706,8 @@ public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedExc registry.add("o2", "object2", null, null); // restart interpreter - factory.restart(factory.getInterpreterSettings(note.getId()).get(0).getId()); - registry = factory.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") + interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId()); + registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess") .getAngularObjectRegistry(); // local and global scope object should be removed @@ -804,7 +807,7 @@ public void testAuthorizationRoles() throws IOException { public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException, IOException { Note note = notebook.createNote(anonymous); - factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); // create three paragraphs Paragraph p1 = note.addParagraph(anonymous); @@ -825,7 +828,7 @@ public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedExc assertEquals(Status.PENDING, p3.getStatus()); // restart interpreter - factory.restart(factory.getInterpreterSettings(note.getId()).get(0).getId()); + interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId()); // make sure three differnt status aborted well. assertEquals(Status.FINISHED, p1.getStatus()); @@ -844,9 +847,9 @@ public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException { p1.setAuthenticationInfo(anonymous); // restart interpreter with per user session enabled - for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) { + for (InterpreterSetting setting : interpreterSettingManager.getInterpreterSettings(note1.getId())) { setting.getOption().setPerNote(setting.getOption().SCOPED); - notebook.getInterpreterFactory().restart(setting.getId()); + notebook.getInterpreterSettingManager().restart(setting.getId()); } note1.run(p1.getId()); @@ -892,9 +895,9 @@ public void testPerSessionInterpreter() throws IOException { // restart interpreter with per note session enabled - for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) { + for (InterpreterSetting setting : notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId())) { setting.getOption().setPerNote(InterpreterOption.SCOPED); - notebook.getInterpreterFactory().restart(setting.getId()); + notebook.getInterpreterSettingManager().restart(setting.getId()); } // run per note session enabled @@ -935,10 +938,10 @@ public void testPerNoteSessionInterpreter() throws IOException { assertEquals(p1.getResult().message().get(0).getData(), p2.getResult().message().get(0).getData()); // restart interpreter with scoped mode enabled - for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) { + for (InterpreterSetting setting : notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId())) { setting.getOption().setPerNote(InterpreterOption.SCOPED); - notebook.getInterpreterFactory().restart(setting.getId(), note1.getId()); - notebook.getInterpreterFactory().restart(setting.getId(), note2.getId()); + notebook.getInterpreterSettingManager().restart(setting.getId(), note1.getId()); + notebook.getInterpreterSettingManager().restart(setting.getId(), note2.getId()); } // run per note session enabled @@ -951,10 +954,10 @@ public void testPerNoteSessionInterpreter() throws IOException { assertNotEquals(p1.getResult().message().get(0).getData(), p2.getResult().message().get(0).getData()); // restart interpreter with isolated mode enabled - for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) { + for (InterpreterSetting setting : notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId())) { setting.getOption().setPerNote(InterpreterOption.ISOLATED); - notebook.getInterpreterFactory().restart(setting.getId(), note1.getId()); - notebook.getInterpreterFactory().restart(setting.getId(), note2.getId()); + notebook.getInterpreterSettingManager().restart(setting.getId(), note1.getId()); + notebook.getInterpreterSettingManager().restart(setting.getId(), note2.getId()); } // run per note process enabled @@ -979,9 +982,9 @@ public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IO p1.setText("getId"); // restart interpreter with per note session enabled - for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) { + for (InterpreterSetting setting : interpreterSettingManager.getInterpreterSettings(note1.getId())) { setting.getOption().setPerNote(InterpreterOption.SCOPED); - notebook.getInterpreterFactory().restart(setting.getId()); + notebook.getInterpreterSettingManager().restart(setting.getId()); } note1.run(p1.getId()); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java index f06a11a9659..69577e963f9 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java @@ -102,7 +102,7 @@ public void should_extract_variable_from_angular_object_registry() throws Except final String scriptBody = "My name is ${name} and I am ${age=20} years old. " + "My occupation is ${ job = engineer | developer | artists}"; - final Paragraph paragraph = new Paragraph(note, null, null); + final Paragraph paragraph = new Paragraph(note, null, null, null); final String paragraphId = paragraph.getId(); final AngularObject nameAO = AngularObjectBuilder.build("name", "DuyHai DOAN", noteId, diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 0da6610a1a5..a77b6462212 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -37,6 +37,7 @@ import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; import org.apache.zeppelin.interpreter.mock.MockInterpreter2; import org.apache.zeppelin.notebook.*; @@ -66,6 +67,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory { private Notebook notebookSync; private NotebookRepoSync notebookRepoSync; private InterpreterFactory factory; + private InterpreterSettingManager interpreterSettingManager; private DependencyResolver depResolver; private SearchService search; private NotebookAuthorization notebookAuthorization; @@ -101,13 +103,14 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); notebookAuthorization = NotebookAuthorization.init(conf); credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath()); - notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this, search, + notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, notebookAuthorization, credentials); anonymous = new AuthenticationInfo("anonymous"); } @@ -236,7 +239,7 @@ public void testOneWaySyncOnReloadedList() throws IOException, SchedulerExceptio System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "true"); conf = ZeppelinConfiguration.create(); notebookRepoSync = new NotebookRepoSync(conf); - notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this, search, + notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, notebookAuthorization, credentials); // check that both storage repos are empty @@ -284,7 +287,7 @@ public void testCheckpointOneStorage() throws IOException, SchedulerException { ZeppelinConfiguration vConf = ZeppelinConfiguration.create(); NotebookRepoSync vRepoSync = new NotebookRepoSync(vConf); - Notebook vNotebookSync = new Notebook(vConf, vRepoSync, schedulerFactory, factory, this, search, + Notebook vNotebookSync = new Notebook(vConf, vRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, notebookAuthorization, credentials); // one git versioned storage initialized diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index 99fea49556a..6db85fee314 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -31,6 +31,7 @@ import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; import org.apache.zeppelin.notebook.JobListenerFactory; import org.apache.zeppelin.notebook.Note; @@ -55,6 +56,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory { private SchedulerFactory schedulerFactory; private Notebook notebook; private NotebookRepo notebookRepo; + private InterpreterSettingManager interpreterSettingManager; private InterpreterFactory factory; private DependencyResolver depResolver; private NotebookAuthorization notebookAuthorization; @@ -84,12 +86,13 @@ public void setUp() throws Exception { this.schedulerFactory = new SchedulerFactory(); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); notebookAuthorization = NotebookAuthorization.init(conf); - notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, this, search, + notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search, notebookAuthorization, null); } @@ -118,7 +121,7 @@ public void testInvalidJsonFile() throws IOException { public void testSaveNotebook() throws IOException, InterruptedException { AuthenticationInfo anonymous = new AuthenticationInfo("anonymous"); Note note = notebook.createNote(anonymous); - factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); Paragraph p1 = note.addParagraph(AuthenticationInfo.ANONYMOUS); Map config = p1.getConfig(); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java index d7bf27b04e7..cf5de756fbb 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java @@ -26,6 +26,7 @@ import java.util.Map; import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.notebook.repo.NotebookRepo; @@ -41,6 +42,7 @@ public class LuceneSearchTest { private static NotebookRepo notebookRepoMock; private static InterpreterFactory interpreterFactory; + private static InterpreterSettingManager interpreterSettingManager; private SearchService noteSearchService; private AuthenticationInfo anonymous; @@ -49,6 +51,7 @@ public class LuceneSearchTest { public static void beforeStartUp() { notebookRepoMock = mock(NotebookRepo.class); interpreterFactory = mock(InterpreterFactory.class); + interpreterSettingManager = mock(InterpreterSettingManager.class); // when(replLoaderMock.getInterpreterSettings()) // .thenReturn(ImmutableList.of()); @@ -288,7 +291,7 @@ private Paragraph addParagraphWithTextAndTitle(Note note, String text, String ti } private Note newNote(String name) { - Note note = new Note(notebookRepoMock, interpreterFactory, null, noteSearchService, null, null); + Note note = new Note(notebookRepoMock, interpreterFactory, interpreterSettingManager, null, noteSearchService, null, null); note.setName(name); return note; }