diff --git a/docs/rest-api/rest-notebook.md b/docs/rest-api/rest-notebook.md index 7393c5a2a87..fade802c5b4 100644 --- a/docs/rest-api/rest-notebook.md +++ b/docs/rest-api/rest-notebook.md @@ -556,4 +556,33 @@ limitations under the License. 
{"status":"OK","body":"* * * * * ?"}
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
Full-text search through the paragraphs in all notebooks
Description```GET``` request will return list of matching paragraphs +
URL```http://[zeppelin-server]:[zeppelin-port]/api/notebook/search?q=[query]```
Success code200
Fail code 500
Sample JSON response 
{"status":"OK", body: [{"id":"/paragraph/", "name":"Notebook Name", "snippet":"", "text":""}]}
diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE index 1cce5dcec9c..52ce3ebc7f2 100644 --- a/zeppelin-distribution/src/bin_license/LICENSE +++ b/zeppelin-distribution/src/bin_license/LICENSE @@ -44,6 +44,7 @@ The following components are provided under Apache License. (Apache 2.0) Apache Tajo (http://tajo.apache.org/) (Apache 2.0) Apache Flink (http://flink.apache.org/) (Apache 2.0) Apache Thrift (http://thrift.apache.org/) + (Apache 2.0) Apache Lucene (https://lucene.apache.org/) (Apache 2.0) Apache Zookeeper (org.apache.zookeeper:zookeeper:jar:3.4.5 - http://zookeeper.apache.org/) (Apache 2.0) Chill (com.twitter:chill-java:jar:0.5.0 - https://github.com/twitter/chill/) (Apache 2.0) Codehaus Plexus (org.codehaus.plexus:plexus:jar:1.5.6 - https://codehaus-plexus.github.io/) @@ -129,10 +130,10 @@ The following components are provided under the MIT License. (The MIT License) Objenesis (org.objenesis:objenesis:2.1 - https://github.com/easymock/objenesis) - Copyright (c) 2006-2015 the original author and authors (The MIT License) JCL 1.1.1 implemented over SLF4J (org.slf4j:jcl-over-slf4j:1.7.5 - http://www.slf4j.org) (The MIT License) JUL to SLF4J bridge (org.slf4j:jul-to-slf4j:1.7.5 - http://www.slf4j.org) + (The MIT License) angular-resource (angular-resource - https://github.com/angular/angular.js/tree/master/src/ngResource) (The MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.4 - https://github.com/ralfstx/minimal-json) - ======================================================================== BSD-style licenses ======================================================================== diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java index 4c8c70a4af6..c803d78f26f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java @@ -37,7 +37,6 @@ * and saving/loading jobs from disk. * Changing/adding/deleting non transitive field name need consideration of that. * - * @author Leemoonsoo */ public abstract class Job { /** diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 0563374da5a..e77ee6ca38d 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -197,6 +197,22 @@ 2.2.1 + + com.sun.jersey + jersey-servlet + 1.13 + + + + javax.ws.rs + javax.ws.rs-api + 2.0-m10 + + + + org.scala-lang + scala-library + org.seleniumhq.selenium @@ -240,23 +256,6 @@ test - - com.sun.jersey - jersey-servlet - 1.13 - - - - javax.ws.rs - javax.ws.rs-api - 2.0-m10 - - - - org.scala-lang - scala-library - - org.scalatest scalatest_2.10 @@ -268,6 +267,7 @@ org.mockito mockito-all 1.9.0 + test diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index d9e7cf70d0c..fb4e99473a7 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -22,19 +22,29 @@ import java.util.List; import java.util.Map; -import javax.ws.rs.*; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import org.apache.commons.lang3.StringUtils; -import org.apache.zeppelin.display.Input; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.Paragraph; -import org.apache.zeppelin.rest.message.*; +import org.apache.zeppelin.rest.message.CronRequest; +import org.apache.zeppelin.rest.message.InterpreterSettingListForNoteBind; +import org.apache.zeppelin.rest.message.NewNotebookRequest; +import org.apache.zeppelin.rest.message.NewParagraphRequest; +import org.apache.zeppelin.rest.message.RunParagraphWithParametersRequest; +import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.server.JsonResponse; -import org.apache.zeppelin.server.ZeppelinServer; import org.apache.zeppelin.socket.NotebookServer; import org.quartz.CronExpression; import org.slf4j.Logger; @@ -49,17 +59,18 @@ @Path("/notebook") @Produces("application/json") public class NotebookRestApi { - Logger logger = LoggerFactory.getLogger(NotebookRestApi.class); + private static final Logger LOG = LoggerFactory.getLogger(NotebookRestApi.class); Gson gson = new Gson(); private Notebook notebook; private NotebookServer notebookServer; + private SearchService notebookIndex; public NotebookRestApi() {} - public NotebookRestApi(Notebook notebook, NotebookServer notebookServer) { - + public NotebookRestApi(Notebook notebook, NotebookServer notebookServer, SearchService search) { this.notebook = notebook; this.notebookServer = notebookServer; + this.notebookIndex = search; } /** @@ -71,7 +82,7 @@ public NotebookRestApi(Notebook notebook, NotebookServer notebookServer) { public Response bind(@PathParam("noteId") String noteId, String req) throws IOException { List settingIdList = gson.fromJson(req, new TypeToken>(){}.getType()); notebook.bindInterpretersToNote(noteId, settingIdList); - return new JsonResponse(Status.OK).build(); + return new JsonResponse<>(Status.OK).build(); } /** @@ -114,14 +125,14 @@ public Response bind(@PathParam("noteId") String noteId) { ); } } - return new JsonResponse(Status.OK, "", settingList).build(); + return new JsonResponse<>(Status.OK, "", settingList).build(); } @GET @Path("/") public Response getNotebookList() throws IOException { List> notesInfo = notebookServer.generateNotebooksInfo(); - return new JsonResponse(Status.OK, "", notesInfo ).build(); + return new JsonResponse<>(Status.OK, "", notesInfo ).build(); } @GET @@ -129,10 +140,10 @@ public Response getNotebookList() throws IOException { public Response getNotebook(@PathParam("notebookId") String notebookId) throws IOException { Note note = notebook.getNote(notebookId); if (note == null) { - return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } - return new JsonResponse(Status.OK, "", note).build(); + return new JsonResponse<>(Status.OK, "", note).build(); } /** @@ -144,7 +155,7 @@ public Response getNotebook(@PathParam("notebookId") String notebookId) throws I @POST @Path("/") public Response createNote(String message) throws IOException { - logger.info("Create new notebook by JSON {}" , message); + LOG.info("Create new notebook by JSON {}" , message); NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class); Note note = notebook.createNote(); @@ -165,7 +176,7 @@ public Response createNote(String message) throws IOException { note.persist(); notebookServer.broadcastNote(note); notebookServer.broadcastNoteList(); - return new JsonResponse(Status.CREATED, "", note.getId() ).build(); + return new JsonResponse<>(Status.CREATED, "", note.getId() ).build(); } /** @@ -177,7 +188,7 @@ public Response createNote(String message) throws IOException { @DELETE @Path("{notebookId}") public Response deleteNote(@PathParam("notebookId") String notebookId) throws IOException { - logger.info("Delete notebook {} ", notebookId); + LOG.info("Delete notebook {} ", notebookId); if (!(notebookId.isEmpty())) { Note note = notebook.getNote(notebookId); if (note != null) { @@ -185,7 +196,7 @@ public Response deleteNote(@PathParam("notebookId") String notebookId) throws IO } } notebookServer.broadcastNoteList(); - return new JsonResponse(Status.OK, "").build(); + return new JsonResponse<>(Status.OK, "").build(); } /** @@ -198,14 +209,14 @@ public Response deleteNote(@PathParam("notebookId") String notebookId) throws IO @Path("{notebookId}") public Response cloneNote(@PathParam("notebookId") String notebookId, String message) throws IOException, CloneNotSupportedException, IllegalArgumentException { - logger.info("clone notebook by JSON {}" , message); + LOG.info("clone notebook by JSON {}" , message); NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class); String newNoteName = request.getName(); Note newNote = notebook.cloneNote(notebookId, newNoteName); notebookServer.broadcastNote(newNote); notebookServer.broadcastNoteList(); - return new JsonResponse(Status.CREATED, "", newNote.getId()).build(); + return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build(); } /** @@ -218,14 +229,14 @@ public Response cloneNote(@PathParam("notebookId") String notebookId, String mes @Path("job/{notebookId}") public Response runNoteJobs(@PathParam("notebookId") String notebookId) throws IOException, IllegalArgumentException { - logger.info("run notebook jobs {} ", notebookId); + LOG.info("run notebook jobs {} ", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { - return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } note.runAll(); - return new JsonResponse(Status.OK).build(); + return new JsonResponse<>(Status.OK).build(); } /** @@ -238,10 +249,10 @@ public Response runNoteJobs(@PathParam("notebookId") String notebookId) throws @Path("job/{notebookId}") public Response stopNoteJobs(@PathParam("notebookId") String notebookId) throws IOException, IllegalArgumentException { - logger.info("stop notebook jobs {} ", notebookId); + LOG.info("stop notebook jobs {} ", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { - return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } for (Paragraph p : note.getParagraphs()) { @@ -249,7 +260,7 @@ public Response stopNoteJobs(@PathParam("notebookId") String notebookId) throws p.abort(); } } - return new JsonResponse(Status.OK).build(); + return new JsonResponse<>(Status.OK).build(); } /** @@ -262,19 +273,21 @@ public Response stopNoteJobs(@PathParam("notebookId") String notebookId) throws @Path("job/{notebookId}") public Response getNoteJobStatus(@PathParam("notebookId") String notebookId) throws IOException, IllegalArgumentException { - logger.info("get notebook job status."); + LOG.info("get notebook job status."); Note note = notebook.getNote(notebookId); if (note == null) { - return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } - return new JsonResponse(Status.OK, null, note.generateParagraphsInfo()).build(); + return new JsonResponse<>(Status.OK, null, note.generateParagraphsInfo()).build(); } /** * Run paragraph job REST API + * * @param message - JSON with params if user wants to update dynamic form's value * null, empty string, empty json if user doesn't want to update + * * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @@ -284,16 +297,16 @@ public Response runParagraph(@PathParam("notebookId") String notebookId, @PathParam("paragraphId") String paragraphId, String message) throws IOException, IllegalArgumentException { - logger.info("run paragraph job {} {} {}", notebookId, paragraphId, message); + LOG.info("run paragraph job {} {} {}", notebookId, paragraphId, message); Note note = notebook.getNote(notebookId); if (note == null) { - return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } Paragraph paragraph = note.getParagraph(paragraphId); if (paragraph == null) { - return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build(); } // handle params if presented @@ -308,7 +321,7 @@ public Response runParagraph(@PathParam("notebookId") String notebookId, } note.run(paragraph.getId()); - return new JsonResponse(Status.OK).build(); + return new JsonResponse<>(Status.OK).build(); } /** @@ -322,18 +335,18 @@ public Response runParagraph(@PathParam("notebookId") String notebookId, public Response stopParagraph(@PathParam("notebookId") String notebookId, @PathParam("paragraphId") String paragraphId) throws IOException, IllegalArgumentException { - logger.info("stop paragraph job {} ", notebookId); + LOG.info("stop paragraph job {} ", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { - return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } Paragraph p = note.getParagraph(paragraphId); if (p == null) { - return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build(); } p.abort(); - return new JsonResponse(Status.OK).build(); + return new JsonResponse<>(Status.OK).build(); } /** @@ -346,18 +359,18 @@ public Response stopParagraph(@PathParam("notebookId") String notebookId, @Path("cron/{notebookId}") public Response registerCronJob(@PathParam("notebookId") String notebookId, String message) throws IOException, IllegalArgumentException { - logger.info("Register cron job note={} request cron msg={}", notebookId, message); + LOG.info("Register cron job note={} request cron msg={}", notebookId, message); CronRequest request = gson.fromJson(message, CronRequest.class); Note note = notebook.getNote(notebookId); if (note == null) { - return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } if (!CronExpression.isValidExpression(request.getCronString())) { - return new JsonResponse(Status.BAD_REQUEST, "wrong cron expressions.").build(); + return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build(); } Map config = note.getConfig(); @@ -365,7 +378,7 @@ public Response registerCronJob(@PathParam("notebookId") String notebookId, Stri note.setConfig(config); notebook.refreshCron(note.id()); - return new JsonResponse(Status.OK).build(); + return new JsonResponse<>(Status.OK).build(); } /** @@ -378,11 +391,11 @@ public Response registerCronJob(@PathParam("notebookId") String notebookId, Stri @Path("cron/{notebookId}") public Response removeCronJob(@PathParam("notebookId") String notebookId) throws IOException, IllegalArgumentException { - logger.info("Remove cron job note {}", notebookId); + LOG.info("Remove cron job note {}", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { - return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } Map config = note.getConfig(); @@ -390,7 +403,7 @@ public Response removeCronJob(@PathParam("notebookId") String notebookId) throws note.setConfig(config); notebook.refreshCron(note.id()); - return new JsonResponse(Status.OK).build(); + return new JsonResponse<>(Status.OK).build(); } /** @@ -403,13 +416,26 @@ public Response removeCronJob(@PathParam("notebookId") String notebookId) throws @Path("cron/{notebookId}") public Response getCronJob(@PathParam("notebookId") String notebookId) throws IOException, IllegalArgumentException { - logger.info("Get cron job note {}", notebookId); + LOG.info("Get cron job note {}", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { - return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } - return new JsonResponse(Status.OK, note.getConfig().get("cron")).build(); + return new JsonResponse<>(Status.OK, note.getConfig().get("cron")).build(); } + + /** + * Search for a Notes + */ + @GET + @Path("search") + public Response search(@QueryParam("q") String queryTerm) { + LOG.info("Searching notebooks for: {}", queryTerm); + List> notebooksFound = notebookIndex.query(queryTerm); + LOG.info("{} notbooks found", notebooksFound.size()); + return new JsonResponse<>(Status.OK, notebooksFound).build(); + } + } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/ZeppelinRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/ZeppelinRestApi.java index 367f9238f46..9a0b883e79b 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/ZeppelinRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/ZeppelinRestApi.java @@ -24,7 +24,6 @@ /** * Zeppelin root rest api endpoint. * - * @author anthonycorbacho * @since 0.3.4 */ @Path("/") diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 7286b355a5c..fd115ee18cb 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -38,6 +38,8 @@ import org.apache.zeppelin.rest.NotebookRestApi; import org.apache.zeppelin.rest.ZeppelinRestApi; import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.apache.zeppelin.search.SearchService; +import org.apache.zeppelin.search.LuceneSearch; import org.apache.zeppelin.socket.NotebookServer; import org.eclipse.jetty.server.AbstractConnector; import org.eclipse.jetty.server.Handler; @@ -69,17 +71,18 @@ public class ZeppelinServer extends Application { private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; private NotebookRepo notebookRepo; + private SearchService notebookIndex; public ZeppelinServer() throws Exception { - LOG.info("Constructor starteds"); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); this.schedulerFactory = new SchedulerFactory(); this.replFactory = new InterpreterFactory(conf, notebookWsServer); this.notebookRepo = new NotebookRepoSync(conf); + this.notebookIndex = new LuceneSearch(); - notebook = new Notebook(conf, notebookRepo, schedulerFactory, replFactory, notebookWsServer); - LOG.info("Constructor finished"); + notebook = new Notebook(conf, + notebookRepo, schedulerFactory, replFactory, notebookWsServer, notebookIndex); } public static void main(String[] args) throws InterruptedException { @@ -264,7 +267,7 @@ public Set getSingletons() { ZeppelinRestApi root = new ZeppelinRestApi(); singletons.add(root); - NotebookRestApi notebookApi = new NotebookRestApi(notebook, notebookWsServer); + NotebookRestApi notebookApi = new NotebookRestApi(notebook, notebookWsServer, notebookIndex); singletons.add(notebookApi); InterpreterRestApi interpreterApi = new InterpreterRestApi(replFactory); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index a010e581ceb..554f68cf073 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -15,6 +15,7 @@ * limitations under the License. */ package org.apache.zeppelin.socket; + import java.io.IOException; import java.net.URISyntaxException; import java.net.UnknownHostException; @@ -49,14 +50,14 @@ import com.google.common.base.Strings; import com.google.gson.Gson; + /** * Zeppelin websocket service. * */ public class NotebookServer extends WebSocketServlet implements NotebookSocketListener, JobListenerFactory, AngularObjectRegistryListener { - private static final Logger LOG = LoggerFactory - .getLogger(NotebookServer.class); + private static final Logger LOG = LoggerFactory.getLogger(NotebookServer.class); Gson gson = new Gson(); final Map> noteSocketMap = new HashMap<>(); final Queue connectedSockets = new ConcurrentLinkedQueue<>(); @@ -64,9 +65,9 @@ public class NotebookServer extends WebSocketServlet implements private Notebook notebook() { return ZeppelinServer.notebook; } + @Override public boolean checkOrigin(HttpServletRequest request, String origin) { - try { return SecurityUtils.isValidOrigin(origin, ZeppelinConfiguration.create()); } catch (UnknownHostException e) { @@ -74,7 +75,6 @@ public boolean checkOrigin(HttpServletRequest request, String origin) { } catch (URISyntaxException e) { e.printStackTrace(); } - return false; } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index caac5a08fe3..2bca9c08593 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -17,9 +17,13 @@ package org.apache.zeppelin.rest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,13 +32,10 @@ import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.httpclient.methods.PutMethod; import org.apache.commons.lang3.StringUtils; -import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Paragraph; -import org.apache.zeppelin.rest.message.NewParagraphRequest; import org.apache.zeppelin.scheduler.Job.Status; -import org.apache.zeppelin.server.JsonResponse; import org.apache.zeppelin.server.ZeppelinServer; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -45,13 +46,9 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import static org.junit.Assert.*; - /** * BASIC Zeppelin rest api tests * - * @author anthonycorbacho - * */ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class ZeppelinRestApiTest extends AbstractTestRestApi { diff --git a/zeppelin-web/bower.json b/zeppelin-web/bower.json index 4e5c3534066..ee9ea466f2e 100644 --- a/zeppelin-web/bower.json +++ b/zeppelin-web/bower.json @@ -11,6 +11,7 @@ "angular-animate": "1.3.8", "angular-touch": "1.3.8", "angular-route": "1.3.8", + "angular-resource": "1.3.8", "angular-bootstrap": "~0.13.0", "angular-websocket": "~1.0.13", "ace-builds": "1.1.9", diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index 64d43bb3c59..92e7345db05 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -32,7 +32,8 @@ angular.module('zeppelinWebApp', [ 'puElasticInput', 'xeditable', 'ngToast', - 'focus-if' + 'focus-if', + 'ngResource' ]) .filter('breakFilter', function() { return function (text) { @@ -50,6 +51,10 @@ angular.module('zeppelinWebApp', [ templateUrl: 'app/notebook/notebook.html', controller: 'NotebookCtrl' }) + .when('/notebook/:noteId/paragraph?=:paragraphId', { + templateUrl: 'app/notebook/notebook.html', + controller: 'NotebookCtrl' + }) .when('/notebook/:noteId/paragraph/:paragraphId?', { templateUrl: 'app/notebook/notebook.html', controller: 'NotebookCtrl' @@ -58,6 +63,10 @@ angular.module('zeppelinWebApp', [ templateUrl: 'app/interpreter/interpreter.html', controller: 'InterpreterCtrl' }) + .when('/search/:searchTerm', { + templateUrl: 'app/search/result-list.html', + controller: 'SearchResultCtrl' + }) .otherwise({ redirectTo: '/' }); diff --git a/zeppelin-web/src/app/notebook/notebook.controller.js b/zeppelin-web/src/app/notebook/notebook.controller.js index 54903ff9b58..55384ff3b0a 100644 --- a/zeppelin-web/src/app/notebook/notebook.controller.js +++ b/zeppelin-web/src/app/notebook/notebook.controller.js @@ -1,4 +1,5 @@ /* jshint loopfunc: true */ +/* global $: false */ /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +15,9 @@ */ 'use strict'; -angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $route, $routeParams, $location, - $rootScope, $http, websocketMsgSrv, baseUrlSrv, - $timeout, SaveAsService) { +angular.module('zeppelinWebApp').controller('NotebookCtrl', + function($scope, $route, $routeParams, $location, $rootScope, $http, + websocketMsgSrv, baseUrlSrv, $timeout, SaveAsService) { $scope.note = null; $scope.showEditor = false; $scope.editorToggled = false; @@ -66,6 +67,26 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro /** Init the new controller */ var initNotebook = function() { websocketMsgSrv.getNotebook($routeParams.noteId); + + var currentRoute = $route.current; + + if (currentRoute) { + + setTimeout( + function() { + var routeParams = currentRoute.params; + var $id = $('#' + routeParams.paragraph + '_container'); + + if ($id.length > 0) { + // adjust for navbar + var top = $id.offset().top - 103; + $('html, body').scrollTo({top: top, left: 0}); + } + + }, + 1000 + ); + } }; initNotebook(); diff --git a/zeppelin-web/src/app/search/result-list.controller.js b/zeppelin-web/src/app/search/result-list.controller.js new file mode 100644 index 00000000000..0d55442e525 --- /dev/null +++ b/zeppelin-web/src/app/search/result-list.controller.js @@ -0,0 +1,119 @@ +/* jshint loopfunc: true */ +/* + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular + .module('zeppelinWebApp') + .controller('SearchResultCtrl', function($scope, $routeParams, searchService) { + + var results = searchService.search({'q': $routeParams.searchTerm}).query(); + + results.$promise.then(function(result) { + $scope.notes = result.body.map(function(note) { + // redirect to notebook when search result is a notebook itself, + // not a paragraph + if (!/\/paragraph\//.test(note.id)) { + return note; + } + + note.id = note.id.replace('paragraph/', '?paragraph=') + + '&term=' + + $routeParams.searchTerm; + + return note; + }); + }); + + $scope.page = 0; + $scope.allResults = false; + + $scope.highlightSearchResults = function(note) { + return function(_editor) { + function getEditorMode(text) { + var editorModes = { + 'ace/mode/scala': /^%spark/, + 'ace/mode/sql': /^%(\w*\.)?\wql/, + 'ace/mode/markdown': /^%md/, + 'ace/mode/sh': /^%sh/ + }; + + return Object.keys(editorModes).reduce(function(res, mode) { + return editorModes[mode].test(text)? mode : res; + }, 'ace/mode/scala'); + } + + var Range = ace.require('ace/range').Range; + + _editor.setOption('highlightActiveLine', false); + _editor.$blockScrolling = Infinity; + _editor.setReadOnly(true); + _editor.renderer.setShowGutter(false); + _editor.setTheme('ace/theme/chrome'); + _editor.getSession().setMode(getEditorMode(note.text)); + + function getIndeces(term) { + return function(str) { + var indeces = []; + var i = -1; + while((i = str.indexOf(term, i + 1)) >= 0) { + indeces.push(i); + } + return indeces; + }; + } + + var lines = note.snippet + .split('\n') + .map(function(line, row) { + var match = line.match(/(.+?)<\/B>/); + + // return early if nothing to highlight + if (!match) { + return line; + } + + var term = match[1]; + var __line = line + .replace(//g, '') + .replace(/<\/B>/g, ''); + + var indeces = getIndeces(term)(__line); + + indeces.forEach(function(start) { + var end = start + term.length; + _editor + .getSession() + .addMarker( + new Range(row, start, row, end), + 'search-results-highlight', + 'line' + ); + }); + + return __line; + }); + + // resize editor based on content length + _editor.setOption( + 'maxLines', + lines.reduce(function(len, line) {return len + line.length;}, 0) + ); + + _editor.getSession().setValue(lines.join('\n')); + + }; + }; + +}); diff --git a/zeppelin-web/src/app/search/result-list.html b/zeppelin-web/src/app/search/result-list.html new file mode 100644 index 00000000000..2d2b6cfc510 --- /dev/null +++ b/zeppelin-web/src/app/search/result-list.html @@ -0,0 +1,42 @@ + +
+
+
+
+
+
diff --git a/zeppelin-web/src/app/search/search.css b/zeppelin-web/src/app/search/search.css new file mode 100644 index 00000000000..e89c765f393 --- /dev/null +++ b/zeppelin-web/src/app/search/search.css @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.search-results { + list-style-type: none; + margin: 10% auto 0; + padding: 0; +} + +.search-result { + height: 200px; +} + +.search-results-header { + text-decoration: none; +} + +.search-results-highlight { + background-color: yellow; + position: absolute; +} + +/* remove error highlighting */ +.search-results .ace_invalid { + background: none !important; +} diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 81b28de6c69..30e6ac27892 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -15,8 +15,7 @@ 'use strict'; angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootScope, $routeParams, - notebookListDataFactory, websocketMsgSrv, - arrayOrderingSrv) { + $location, notebookListDataFactory, websocketMsgSrv, arrayOrderingSrv) { /** Current list of notes (ids) */ var vm = this; @@ -35,6 +34,19 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco vm.connected = param; }); + $rootScope.$on('$locationChangeSuccess', function () { + var path = $location.path(); + // hacky solution to clear search bar + // TODO(felizbear): figure out how to make ng-click work in navbar + if (path === '/') { + $scope.searchTerm = ''; + } + }); + + $scope.search = function() { + $location.url(/search/ + $scope.searchTerm); + }; + function loadNotes() { websocketMsgSrv.getNotebookList(); } diff --git a/zeppelin-web/src/components/navbar/navbar.html b/zeppelin-web/src/components/navbar/navbar.html index 82210da1855..86a85122add 100644 --- a/zeppelin-web/src/components/navbar/navbar.html +++ b/zeppelin-web/src/components/navbar/navbar.html @@ -43,7 +43,34 @@ Interpreter + +
    +
  • + +
    +
    + + + + +
    +
    +
  • Connected diff --git a/zeppelin-web/src/components/searchService/search.service.js b/zeppelin-web/src/components/searchService/search.service.js new file mode 100644 index 00000000000..920b5e08a52 --- /dev/null +++ b/zeppelin-web/src/components/searchService/search.service.js @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +angular.module('zeppelinWebApp').service('searchService', function($resource, baseUrlSrv) { + + this.search = function(term) { + console.log('Searching for: %o', term.q); + if (!term.q) { //TODO(bzz): empty string check + return; + } + var encQuery = window.encodeURIComponent(term.q); + return $resource(baseUrlSrv.getRestApiBase()+'/notebook/search?q='+encQuery, {}, { + query: {method:'GET'} + }); + }; + +}); diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index 4bddd9849d4..2b114654355 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -49,6 +49,7 @@ + @@ -95,6 +96,7 @@ + @@ -131,6 +133,7 @@ + @@ -147,6 +150,7 @@ + diff --git a/zeppelin-web/test/karma.conf.js b/zeppelin-web/test/karma.conf.js index 1cd232878ea..34895b4d481 100644 --- a/zeppelin-web/test/karma.conf.js +++ b/zeppelin-web/test/karma.conf.js @@ -29,6 +29,7 @@ module.exports = function(config) { 'bower_components/angular-animate/angular-animate.js', 'bower_components/angular-touch/angular-touch.js', 'bower_components/angular-route/angular-route.js', + 'bower_components/angular-resource/angular-resource.js', 'bower_components/angular-bootstrap/ui-bootstrap-tpls.js', 'bower_components/angular-websocket/angular-websocket.min.js', 'bower_components/ace-builds/src-noconflict/ace.js', diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index e9de7482078..0ef60bf3a69 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -123,6 +123,36 @@ guava + + org.apache.lucene + lucene-core + 5.3.1 + + + + org.apache.lucene + lucene-analyzers-common + 5.3.1 + + + + org.apache.lucene + lucene-queryparser + 5.3.1 + + + + org.apache.lucene + lucene-highlighter + 5.3.1 + + + + junit + junit + test + + org.reflections reflections diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 93dcec582ae..6a3074f5889 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -20,12 +20,12 @@ import java.io.IOException; import java.io.Serializable; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; -import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.Input; @@ -39,6 +39,7 @@ import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.JobListener; +import org.apache.zeppelin.search.SearchService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,40 +50,43 @@ */ public class Note implements Serializable, JobListener { transient Logger logger = LoggerFactory.getLogger(Note.class); - List paragraphs = new LinkedList(); - private String name; + private static final long serialVersionUID = 7920699076577612429L; + + List paragraphs = new LinkedList<>(); + private String name = ""; private String id; - Map> angularObjects = new HashMap>(); + @SuppressWarnings("rawtypes") + Map> angularObjects = new HashMap<>(); private transient NoteInterpreterLoader replLoader; - private transient ZeppelinConfiguration conf; private transient JobListenerFactory jobListenerFactory; private transient NotebookRepo repo; + private transient SearchService index; /** * note configurations. * * - looknfeel - cron */ - private Map config = new HashMap(); + private Map config = new HashMap<>(); /** * note information. * * - cron : cron expression validity. */ - private Map info = new HashMap(); + private Map info = new HashMap<>(); public Note() {} - public Note(NotebookRepo repo, - NoteInterpreterLoader replLoader, - JobListenerFactory jobListenerFactory) { + public Note(NotebookRepo repo, NoteInterpreterLoader replLoader, + JobListenerFactory jlFactory, SearchService noteIndex) { this.repo = repo; this.replLoader = replLoader; - this.jobListenerFactory = jobListenerFactory; + this.jobListenerFactory = jlFactory; + this.index = noteIndex; generateId(); } @@ -130,6 +134,11 @@ public void setNotebookRepo(NotebookRepo repo) { this.repo = repo; } + public void setIndex(SearchService index) { + this.index = index; + } + + @SuppressWarnings("rawtypes") public Map> getAngularObjects() { return angularObjects; } @@ -193,14 +202,16 @@ public Paragraph insertParagraph(int index) { * Remove paragraph by id. * * @param paragraphId - * @return + * @return a paragraph that was deleted, or null otherwise */ public Paragraph removeParagraph(String paragraphId) { synchronized (paragraphs) { - for (int i = 0; i < paragraphs.size(); i++) { - Paragraph p = paragraphs.get(i); + Iterator i = paragraphs.iterator(); + while (i.hasNext()) { + Paragraph p = i.next(); if (p.getId().equals(paragraphId)) { - paragraphs.remove(i); + index.deleteIndexDoc(this, p); + i.remove(); return p; } } @@ -293,7 +304,7 @@ public Paragraph getLastParagraph() { return paragraphs.get(paragraphs.size() - 1); } } - + public List> generateParagraphsInfo (){ List> paragraphsInfo = new LinkedList<>(); synchronized (paragraphs) { @@ -307,7 +318,7 @@ public List> generateParagraphsInfo (){ } } return paragraphsInfo; - } + } /** * Run all paragraphs sequentially. @@ -357,7 +368,7 @@ public List getParagraphs() { } private void snapshotAngularObjectRegistry() { - angularObjects = new HashMap>(); + angularObjects = new HashMap<>(); List settings = replLoader.getInterpreterSettings(); if (settings == null || settings.size() == 0) { @@ -373,6 +384,7 @@ private void snapshotAngularObjectRegistry() { public void persist() throws IOException { snapshotAngularObjectRegistry(); + index.updateIndexDoc(this); repo.save(this); } @@ -382,7 +394,7 @@ public void unpersist() throws IOException { public Map getConfig() { if (config == null) { - config = new HashMap(); + config = new HashMap<>(); } return config; } @@ -393,7 +405,7 @@ public void setConfig(Map config) { public Map getInfo() { if (info == null) { - info = new HashMap(); + info = new HashMap<>(); } return info; } @@ -404,17 +416,10 @@ public void setInfo(Map info) { @Override public void beforeStatusChange(Job job, Status before, Status after) { - Paragraph p = (Paragraph) job; } @Override public void afterStatusChange(Job job, Status before, Status after) { - Paragraph p = (Paragraph) job; - } - - private static Logger logger() { - Logger logger = LoggerFactory.getLogger(Note.class); - return logger; } @Override diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 481f70838b0..c98f2fb3fab 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -27,6 +27,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; @@ -38,6 +39,7 @@ import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.apache.zeppelin.search.SearchService; import org.quartz.CronScheduleBuilder; import org.quartz.CronTrigger; import org.quartz.JobBuilder; @@ -56,7 +58,10 @@ */ public class Notebook { Logger logger = LoggerFactory.getLogger(Notebook.class); + + @SuppressWarnings("unused") @Deprecated //TODO(bzz): remove unused private SchedulerFactory schedulerFactory; + private InterpreterFactory replFactory; /** Keep the order. */ Map notes = new LinkedHashMap(); @@ -65,22 +70,45 @@ public class Notebook { private org.quartz.Scheduler quartzSched; private JobListenerFactory jobListenerFactory; private NotebookRepo notebookRepo; + private SearchService notebookIndex; + /** + * Main constructor \w manual Dependency Injection + * + * @param conf + * @param notebookRepo + * @param schedulerFactory + * @param replFactory + * @param jobListenerFactory + * @param notebookIndex - (nullable) for indexing all notebooks on creating. + * + * @throws IOException + * @throws SchedulerException + */ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, SchedulerFactory schedulerFactory, - InterpreterFactory replFactory, JobListenerFactory jobListenerFactory) throws IOException, - SchedulerException { + InterpreterFactory replFactory, JobListenerFactory jobListenerFactory, + SearchService notebookIndex) throws IOException, SchedulerException { this.conf = conf; this.notebookRepo = notebookRepo; this.schedulerFactory = schedulerFactory; this.replFactory = replFactory; this.jobListenerFactory = jobListenerFactory; + this.notebookIndex = notebookIndex; quertzSchedFact = new org.quartz.impl.StdSchedulerFactory(); quartzSched = quertzSchedFact.getScheduler(); quartzSched.start(); CronJob.notebook = this; loadAllNotes(); + if (this.notebookIndex != null) { + long start = System.nanoTime(); + logger.info("Notebook indexing started..."); + notebookIndex.addIndexDocs(notes.values()); + logger.info("Notebook indexing finished: {} indexed in {}s", notes.size(), + TimeUnit.NANOSECONDS.toSeconds(start - System.nanoTime())); + } + } /** @@ -90,11 +118,14 @@ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, * @throws IOException */ public Note createNote() throws IOException { + Note note; if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) { - return createNote(replFactory.getDefaultInterpreterSettingList()); + note = createNote(replFactory.getDefaultInterpreterSettingList()); } else { - return createNote(null); + note = createNote(null); } + notebookIndex.addIndexDoc(note); + return note; } /** @@ -105,7 +136,7 @@ public Note createNote() throws IOException { */ public Note createNote(List interpreterIds) throws IOException { NoteInterpreterLoader intpLoader = new NoteInterpreterLoader(replFactory); - Note note = new Note(notebookRepo, intpLoader, jobListenerFactory); + Note note = new Note(notebookRepo, intpLoader, jobListenerFactory, notebookIndex); intpLoader.setNoteId(note.id()); synchronized (notes) { notes.put(note.id(), note); @@ -114,6 +145,7 @@ public Note createNote(List interpreterIds) throws IOException { bindInterpretersToNote(note.id(), interpreterIds); } + notebookIndex.addIndexDoc(note); note.persist(); return note; } @@ -144,6 +176,8 @@ public Note cloneNote(String sourceNoteID, String newNoteName) throws for (Paragraph p : paragraphs) { newNote.addCloneParagraph(p); } + + notebookIndex.addIndexDoc(newNote); newNote.persist(); return newNote; } @@ -183,9 +217,11 @@ public Note getNote(String id) { public void removeNote(String id) { Note note; + synchronized (notes) { note = notes.remove(id); } + notebookIndex.deleteIndexDocs(note); // remove from all interpreter instance's angular object registry for (InterpreterSetting settings : replFactory.get()) { @@ -204,6 +240,7 @@ public void removeNote(String id) { } } + @SuppressWarnings("rawtypes") private Note loadNoteFromRepo(String id) { Note note = null; try { @@ -215,20 +252,17 @@ private Note loadNoteFromRepo(String id) { return null; } - // set NoteInterpreterLoader - NoteInterpreterLoader noteInterpreterLoader = new NoteInterpreterLoader( - replFactory); - note.setReplLoader(noteInterpreterLoader); - noteInterpreterLoader.setNoteId(note.id()); + //Manually inject ALL dependencies, as DI constructor was NOT used + note.setIndex(this.notebookIndex); - // set JobListenerFactory - note.setJobListenerFactory(jobListenerFactory); + NoteInterpreterLoader replLoader = new NoteInterpreterLoader(replFactory); + note.setReplLoader(replLoader); + replLoader.setNoteId(note.id()); - // set notebookRepo + note.setJobListenerFactory(jobListenerFactory); note.setNotebookRepo(notebookRepo); - Map angularObjectSnapshot = - new HashMap(); + Map angularObjectSnapshot = new HashMap<>(); // restore angular object -------------- Date lastUpdatedDate = new Date(0); @@ -246,15 +280,11 @@ private Note loadNoteFromRepo(String id) { for (String intpGroupName : savedObjects.keySet()) { List objectList = savedObjects.get(intpGroupName); - for (AngularObject savedObject : objectList) { - SnapshotAngularObject snapshot = angularObjectSnapshot.get(savedObject.getName()); + for (AngularObject object : objectList) { + SnapshotAngularObject snapshot = angularObjectSnapshot.get(object.getName()); if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) { - angularObjectSnapshot.put( - savedObject.getName(), - new SnapshotAngularObject( - intpGroupName, - savedObject, - lastUpdatedDate)); + angularObjectSnapshot.put(object.getName(), + new SnapshotAngularObject(intpGroupName, object, lastUpdatedDate)); } } } @@ -310,6 +340,7 @@ private void reloadAllNotes() throws IOException { } } + @SuppressWarnings("rawtypes") class SnapshotAngularObject { String intpGroupId; AngularObject angularObject; @@ -344,12 +375,9 @@ public List getAllNotes() { } synchronized (notes) { List noteList = new ArrayList(notes.values()); - Collections.sort(noteList, new Comparator() { + Collections.sort(noteList, new Comparator() { @Override - public int compare(Object one, Object two) { - Note note1 = (Note) one; - Note note2 = (Note) two; - + public int compare(Note note1, Note note2) { String name1 = note1.id(); if (note1.getName() != null) { name1 = note1.getName(); @@ -358,7 +386,6 @@ public int compare(Object one, Object two) { if (note2.getName() != null) { name2 = note2.getName(); } - ((Note) one).getName(); return name1.compareTo(name2); } }); @@ -459,6 +486,7 @@ public ZeppelinConfiguration getConf() { public void close() { this.notebookRepo.close(); + this.notebookIndex.close(); } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index ec47efdde61..433095b379c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -23,7 +23,6 @@ import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.Interpreter.FormType; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.InterpreterResult.Type; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.JobListener; import org.slf4j.Logger; @@ -35,10 +34,10 @@ /** * Paragraph is a representation of an execution unit. * - * @author Leemoonsoo */ public class Paragraph extends Job implements Serializable, Cloneable { - private static final transient long serialVersionUID = -6328572073497992016L; + private static final long serialVersionUID = -6328572073497992016L; + private transient NoteInterpreterLoader replLoader; private transient Note note; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java index 08156c7b0ed..a5bf6b3dfdd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java @@ -46,11 +46,11 @@ public class NotebookRepoSync implements NotebookRepo { private List repos = new ArrayList(); /** + * @param noteIndex * @param (conf) * @throws - Exception */ public NotebookRepoSync(ZeppelinConfiguration conf) throws Exception { - config = conf; String allStorageClassNames = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE).trim(); @@ -134,20 +134,18 @@ public void remove(String noteId) throws IOException { } /** - * copy new/updated notes from source to destination storage + * Copies new/updated notes from source to destination storage + * * @throws IOException */ void sync(int sourceRepoIndex, int destRepoIndex) throws IOException { LOG.info("Sync started"); - NotebookRepo sourceRepo = getRepo(sourceRepoIndex); - NotebookRepo destRepo = getRepo(destRepoIndex); - List sourceNotes = sourceRepo.list(); - List destNotes = destRepo.list(); - - Map> noteIDs = notesCheckDiff(sourceNotes, - sourceRepo, - destNotes, - destRepo); + NotebookRepo srcRepo = getRepo(sourceRepoIndex); + NotebookRepo dstRepo = getRepo(destRepoIndex); + List srcNotes = srcRepo.list(); + List dstNotes = dstRepo.list(); + + Map> noteIDs = notesCheckDiff(srcNotes, srcRepo, dstNotes, dstRepo); List pushNoteIDs = noteIDs.get(pushKey); List pullNoteIDs = noteIDs.get(pullKey); if (!pushNoteIDs.isEmpty()) { @@ -155,7 +153,7 @@ void sync(int sourceRepoIndex, int destRepoIndex) throws IOException { for (String id : pushNoteIDs) { LOG.info("ID : " + id); } - pushNotes(pushNoteIDs, sourceRepo, destRepo); + pushNotes(pushNoteIDs, srcRepo, dstRepo); } else { LOG.info("Nothing to push"); } @@ -165,7 +163,7 @@ void sync(int sourceRepoIndex, int destRepoIndex) throws IOException { for (String id : pullNoteIDs) { LOG.info("ID : " + id); } - pushNotes(pullNoteIDs, destRepo, sourceRepo); + pushNotes(pullNoteIDs, dstRepo, srcRepo); } else { LOG.info("Nothing to pull"); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java new file mode 100644 index 00000000000..7f9cbbdd9db --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.search; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.search.highlight.Highlighter; +import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; +import org.apache.lucene.search.highlight.QueryScorer; +import org.apache.lucene.search.highlight.SimpleHTMLFormatter; +import org.apache.lucene.search.highlight.TextFragment; +import org.apache.lucene.search.highlight.TokenSources; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Paragraph; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; + +/** + * Search (both, indexing and query) the notebooks using Lucene. + * + * Query is thread-safe, as creates new IndexReader every time. + * Index is thread-safe, as re-uses single IndexWriter, which is thread-safe. + */ +public class LuceneSearch implements SearchService { + private static final Logger LOG = LoggerFactory.getLogger(LuceneSearch.class); + + private static final String SEARCH_FIELD = "contents"; + static final String PARAGRAPH = "paragraph"; + static final String ID_FIELD = "id"; + + Directory ramDirectory; + Analyzer analyzer; + IndexWriterConfig iwc; + IndexWriter writer; + + public LuceneSearch() { + ramDirectory = new RAMDirectory(); + analyzer = new StandardAnalyzer(); + iwc = new IndexWriterConfig(analyzer); + try { + writer = new IndexWriter(ramDirectory, iwc); + } catch (IOException e) { + LOG.error("Failed to reate new IndexWriter", e); + } + } + + /* (non-Javadoc) + * @see org.apache.zeppelin.search.Search#query(java.lang.String) + */ + @Override + public List> query(String queryStr) { + if (null == ramDirectory) { + throw new IllegalStateException( + "Something went wrong on instance creation time, index dir is null"); + } + List> result = Collections.emptyList(); + try (IndexReader indexReader = DirectoryReader.open(ramDirectory)) { + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + Analyzer analyzer = new StandardAnalyzer(); + QueryParser parser = new QueryParser(SEARCH_FIELD, analyzer); + + Query query = parser.parse(queryStr); + LOG.debug("Searching for: " + query.toString(SEARCH_FIELD)); + + SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter(); + Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(query)); + + result = doSearch(indexSearcher, query, analyzer, highlighter); + indexReader.close(); + } catch (IOException e) { + LOG.error("Failed to open index dir {}, make sure indexing finished OK", ramDirectory, e); + } catch (ParseException e) { + LOG.error("Failed to parse query " + queryStr, e); + } + return result; + } + + private List> doSearch(IndexSearcher searcher, Query query, + Analyzer analyzer, Highlighter highlighter) { + List> matchingParagraphs = Lists.newArrayList(); + ScoreDoc[] hits; + try { + hits = searcher.search(query, 20).scoreDocs; + for (int i = 0; i < hits.length; i++) { + LOG.debug("doc={} score={}", hits[i].doc, hits[i].score); + + int id = hits[i].doc; + Document doc = searcher.doc(id); + String path = doc.get(ID_FIELD); + if (path != null) { + LOG.debug((i + 1) + ". " + path); + String title = doc.get("title"); + if (title != null) { + LOG.debug(" Title: {}", doc.get("title")); + } + + String text = doc.get(SEARCH_FIELD); + TokenStream tokenStream = TokenSources.getTokenStream(searcher.getIndexReader(), id, + SEARCH_FIELD, analyzer); + TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, true, 3); + LOG.debug(" {} fragments found for query '{}'", frag.length, query); + for (int j = 0; j < frag.length; j++) { + if ((frag[j] != null) && (frag[j].getScore() > 0)) { + LOG.debug(" Fragment: {}", frag[j].toString()); + } + } + String fragment = (frag != null && frag.length > 0) ? frag[0].toString() : ""; + + matchingParagraphs.add(ImmutableMap.of("id", path, // /paragraph/ + "name", title, "snippet", fragment, "text", text)); + } else { + LOG.info("{}. No {} for this document", i + 1, ID_FIELD); + } + } + } catch (IOException | InvalidTokenOffsetsException e) { + LOG.error("Exception on searching for {}", query, e); + } + return matchingParagraphs; + } + + /* (non-Javadoc) + * @see org.apache.zeppelin.search.Search#updateIndexDoc(org.apache.zeppelin.notebook.Note) + */ + @Override + public void updateIndexDoc(Note note) throws IOException { + updateIndexNoteName(note); + for (Paragraph p: note.getParagraphs()) { + updateIndexParagraph(note, p); + } + } + + private void updateIndexNoteName(Note note) throws IOException { + String noteName = note.getName(); + String noteId = note.getId(); + LOG.debug("Indexing Notebook {}, '{}'", noteId, noteName); + if (null == noteName || noteName.isEmpty()) { + LOG.debug("Skipping empty notebook name"); + return; + } + updateDoc(noteId, noteName, null); + } + + private void updateIndexParagraph(Note note, Paragraph p) throws IOException { + if (p.getText() == null) { + LOG.debug("Skipping empty paragraph"); + return; + } + updateDoc(note.getId(), note.getName(), p); + } + + /** + * Updates index for the given note: either note.name or a paragraph If + * paragraph is null - updates only for the note.name + * + * @param noteId + * @param noteName + * @param p + * @throws IOException + */ + private void updateDoc(String noteId, String noteName, Paragraph p) throws IOException { + String id = formatId(noteId, p); + Document doc = newDocument(id, noteName, p); + try { + writer.updateDocument(new Term(ID_FIELD, id), doc); + writer.commit(); + } catch (IOException e) { + LOG.error("Failed to updaet index of notebook {}", noteId, e); + } + } + + /** + * If paragraph is not null, id is /paragraphs/, + * otherwise it's just . + */ + static String formatId(String noteId, Paragraph p) { + String id = noteId; + if (null != p) { + id = Joiner.on('/').join(id, PARAGRAPH, p.getId()); + } + return id; + } + + static String formatDeleteId(String noteId, Paragraph p) { + String id = noteId; + if (null != p) { + id = Joiner.on('/').join(id, PARAGRAPH, p.getId()); + } else { + id = id + "*"; + } + return id; + } + + /** + * If paragraph is not null, indexes code in the paragraph, otherwise indexes + * the notebook name. + * + * @param id id of the document, different for Note name and paragraph + * @param noteName name of the note + * @param p paragraph + * @return + */ + private Document newDocument(String id, String noteName, Paragraph p) { + Document doc = new Document(); + + Field pathField = new StringField(ID_FIELD, id, Field.Store.YES); + doc.add(pathField); + doc.add(new StringField("title", noteName, Field.Store.YES)); + + if (null != p) { + doc.add(new TextField(SEARCH_FIELD, p.getText(), Field.Store.YES)); + Date date = p.getDateStarted() != null ? p.getDateStarted() : p.getDateCreated(); + doc.add(new LongField("modified", date.getTime(), Field.Store.NO)); + } else { + doc.add(new TextField(SEARCH_FIELD, noteName, Field.Store.YES)); + } + return doc; + } + + /* (non-Javadoc) + * @see org.apache.zeppelin.search.Search#addIndexDocs(java.util.Collection) + */ + @Override + public void addIndexDocs(Collection collection) { + int docsIndexed = 0; + long start = System.nanoTime(); + try { + for (Note note : collection) { + addIndexDocAsync(note); + docsIndexed++; + } + } catch (IOException e) { + LOG.error("Failed to index all Notebooks", e); + } finally { + try { // save what's been indexed, even if not full collection + writer.commit(); + } catch (IOException e) { + LOG.error("Failed to save index", e); + } + long end = System.nanoTime(); + LOG.info("Indexing {} notebooks took {}ms", docsIndexed, + TimeUnit.NANOSECONDS.toMillis(end - start)); + } + } + + /* (non-Javadoc) + * @see org.apache.zeppelin.search.Search#addIndexDoc(org.apache.zeppelin.notebook.Note) + */ + @Override + public void addIndexDoc(Note note) { + try { + addIndexDocAsync(note); + writer.commit(); + } catch (IOException e) { + LOG.error("Failed to add note {} to index", note, e); + } + } + + /** + * Indexes the given notebook, but does not commit changes. + * + * @param note + * @throws IOException + */ + private void addIndexDocAsync(Note note) throws IOException { + indexNoteName(writer, note.getId(), note.getName()); + for (Paragraph doc : note.getParagraphs()) { + if (doc.getText() == null) { + LOG.debug("Skipping empty paragraph"); + continue; + } + indexDoc(writer, note.getId(), note.getName(), doc); + } + } + + /* (non-Javadoc) + * @see org.apache.zeppelin.search.Search#deleteIndexDocs(org.apache.zeppelin.notebook.Note) + */ + @Override + public void deleteIndexDocs(Note note) { + deleteDoc(note, null); + } + + /* (non-Javadoc) + * @see org.apache.zeppelin.search.Search + * #deleteIndexDoc(org.apache.zeppelin.notebook.Note, org.apache.zeppelin.notebook.Paragraph) + */ + @Override + public void deleteIndexDoc(Note note, Paragraph p) { + deleteDoc(note, p); + } + + private void deleteDoc(Note note, Paragraph p) { + if (null == note) { + LOG.error("Trying to delete note by reference to NULL"); + return; + } + String fullNoteOrJustParagraph = formatDeleteId(note.getId(), p); + LOG.debug("Deleting note {}, out of: {}", note.getId(), writer.numDocs()); + try { + writer.deleteDocuments(new WildcardQuery(new Term(ID_FIELD, fullNoteOrJustParagraph))); + writer.commit(); + } catch (IOException e) { + LOG.error("Failed to delete {} from index by '{}'", note, fullNoteOrJustParagraph, e); + } + LOG.debug("Done, index contains {} docs now" + writer.numDocs()); + } + + /* (non-Javadoc) + * @see org.apache.zeppelin.search.Search#close() + */ + @Override + public void close() { + try { + writer.close(); + } catch (IOException e) { + LOG.error("Failed to .close() the notebook index", e); + } + } + + /** + * Indexes a notebook name + * + * @throws IOException + */ + private void indexNoteName(IndexWriter w, String noteId, String noteName) throws IOException { + LOG.debug("Indexing Notebook {}, '{}'", noteId, noteName); + if (null == noteName || noteName.isEmpty()) { + LOG.debug("Skipping empty notebook name"); + return; + } + indexDoc(w, noteId, noteName, null); + } + + /** + * Indexes a single document: + * - code of the paragraph (if non-null) + * - or just a note name + */ + private void indexDoc(IndexWriter w, String noteId, String noteName, Paragraph p) + throws IOException { + String id = formatId(noteId, p); + Document doc = newDocument(id, noteName, p); + w.addDocument(doc); + } + +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/SearchService.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/SearchService.java new file mode 100644 index 00000000000..64f2b758be2 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/SearchService.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.search; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Paragraph; + +/** + * Search (both, indexing and query) the notebooks. + * + * Intended to have multiple implementation, i.e: + * - local Lucene (in-memory, on-disk) + * - remote Elasticsearch + */ +public interface SearchService { + + /** + * Full-text search in all the notebooks + * + * @param queryStr a query + * @return A list of matching paragraphs (id, text, snippet w/ highlight) + */ + public List> query(String queryStr); + + /** + * Updates all documents in index for the given note: + * - name + * - all paragraphs + * + * @param note a Note to update index for + * @throws IOException + */ + public void updateIndexDoc(Note note) throws IOException; + + /** + * Indexes full collection of notes: all the paragraphs + Note names + * + * @param collection of Notes + */ + public void addIndexDocs(Collection collection); + + /** + * Indexes the given notebook. + * + * @throws IOException If there is a low-level I/O error + */ + public void addIndexDoc(Note note); + + /** + * Deletes all docs on given Note from index + */ + public void deleteIndexDocs(Note note); + + /** + * Deletes doc for a given + * + * @param note + * @param p + * @throws IOException + */ + public void deleteIndexDoc(Note note, Paragraph p); + + /** + * Frees the recourses used by index + */ + public void close(); + +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index ee35773b12f..e2d1aaca330 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; import java.io.File; import java.io.IOException; @@ -45,6 +46,8 @@ import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.JobListener; import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.apache.zeppelin.search.SearchService; +import org.apache.zeppelin.search.LuceneSearch; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -84,8 +87,9 @@ public void setUp() throws Exception { factory = new InterpreterFactory(conf, new InterpreterOption(false), null); + SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); - notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, this); + notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, this, search); } @After @@ -170,7 +174,8 @@ public void testPersist() throws IOException, SchedulerException{ p1.setText("hello world"); note.persist(); - Notebook notebook2 = new Notebook(conf, notebookRepo, schedulerFactory, new InterpreterFactory(conf, null), this); + Notebook notebook2 = new Notebook( + conf, notebookRepo, schedulerFactory, new InterpreterFactory(conf, null), this, null); assertEquals(1, notebook2.getAllNotes().size()); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 64d9b32444d..6d8c50dd1aa 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; import java.io.File; import java.io.IOException; @@ -39,6 +40,8 @@ import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.JobListener; import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.apache.zeppelin.search.SearchService; +import org.apache.zeppelin.search.LuceneSearch; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -86,8 +89,9 @@ public void setUp() throws Exception { factory = new InterpreterFactory(conf, new InterpreterOption(false), null); + SearchService search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); - notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this); + notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this, search); } @After diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index c3bb3a0cdfb..80d1174e2f9 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -18,6 +18,7 @@ package org.apache.zeppelin.notebook.repo; import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; import java.io.File; import java.io.IOException; @@ -32,20 +33,19 @@ import org.apache.zeppelin.notebook.JobListenerFactory; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Notebook; -import org.apache.zeppelin.notebook.NotebookTest; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.scheduler.JobListener; import org.apache.zeppelin.scheduler.SchedulerFactory; -import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.search.SearchService; +import org.apache.zeppelin.search.LuceneSearch; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class VFSNotebookRepoTest implements JobListenerFactory{ - private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class); - +public class VFSNotebookRepoTest implements JobListenerFactory { + private static final Logger LOG = LoggerFactory.getLogger(VFSNotebookRepoTest.class); private ZeppelinConfiguration conf; private SchedulerFactory schedulerFactory; private Notebook notebook; @@ -53,16 +53,15 @@ public class VFSNotebookRepoTest implements JobListenerFactory{ private InterpreterFactory factory; private File mainZepDir; - private File mainNotebookDir; @Before public void setUp() throws Exception { - String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis(); + String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); mainZepDir = new File(zpath); mainZepDir.mkdirs(); new File(mainZepDir, "conf").mkdirs(); - String mainNotePath = zpath+"/notebook"; + String mainNotePath = zpath + "/notebook"; mainNotebookDir = new File(mainNotePath); mainNotebookDir.mkdirs(); @@ -79,15 +78,15 @@ public void setUp() throws Exception { this.schedulerFactory = new SchedulerFactory(); factory = new InterpreterFactory(conf, new InterpreterOption(false), null); + SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); - notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, this); + notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, this, search); } @After public void tearDown() throws Exception { - //FileUtils.deleteDirectory(mainZepDir); if (!FileUtils.deleteQuietly(mainZepDir)) { - logger.error("Failed to delete {} ", mainZepDir.getName()); + LOG.error("Failed to delete {} ", mainZepDir.getName()); } } @@ -97,7 +96,7 @@ public void testSaveNotebook() throws IOException, InterruptedException { note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); Paragraph p1 = note.addParagraph(); - Map config = p1.getConfig(); + Map config = p1.getConfig(); config.put("enabled", true); p1.setConfig(config); p1.setText("%mock1 hello world"); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java new file mode 100644 index 00000000000..f74d95eb375 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.search; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.*; +import static org.apache.zeppelin.search.LuceneSearch.formatId; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInterpreterLoader; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.notebook.repo.NotebookRepo; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; + +public class LuceneSearchTest { + + private static NoteInterpreterLoader replLoaderMock; + private static NotebookRepo notebookRepoMock; + private SearchService notebookIndex; + + @BeforeClass + public static void beforeStartUp() { + notebookRepoMock = mock(NotebookRepo.class); + replLoaderMock = mock(NoteInterpreterLoader.class); + + when(replLoaderMock.getInterpreterSettings()) + .thenReturn(ImmutableList.of()); + } + + @Before + public void startUp() { + notebookIndex = new LuceneSearch(); + } + + @After + public void shutDown() { + notebookIndex.close(); + } + + @Test public void canIndexNotebook() { + //give + Note note1 = newNoteWithParapgraph("Notebook1", "test"); + Note note2 = newNoteWithParapgraph("Notebook2", "not test"); + List notebook = Arrays.asList(note1, note2); + + //when + notebookIndex.addIndexDocs(notebook); + } + + @Test public void canIndexAndQuery() { + //given + Note note1 = newNoteWithParapgraph("Notebook1", "test"); + Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); + + //when + List> results = notebookIndex.query("all"); + + //then + assertThat(results).isNotEmpty(); + assertThat(results.size()).isEqualTo(1); + assertThat(results.get(0)) + .containsEntry("id", formatId(note2.getId(), note2.getLastParagraph())); + } + + @Test public void canIndexAndQueryByNotebookName() { + //given + Note note1 = newNoteWithParapgraph("Notebook1", "test"); + Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); + + //when + List> results = notebookIndex.query("Notebook1"); + + //then + assertThat(results).isNotEmpty(); + assertThat(results.size()).isEqualTo(1); + assertThat(results.get(0)).containsEntry("id", note1.getId()); + } + + @Test public void indexKeyContract() throws IOException { + //give + Note note1 = newNoteWithParapgraph("Notebook1", "test"); + //when + notebookIndex.addIndexDoc(note1); + //then + String id = resultForQuery("test").get(0).get(LuceneSearch.ID_FIELD); + + assertThat(Splitter.on("/").split(id)) //key structure /paragraph/ + .containsAllOf(note1.getId(), LuceneSearch.PARAGRAPH, note1.getLastParagraph().getId()); + } + + @Test //(expected=IllegalStateException.class) + public void canNotSearchBeforeIndexing() { + //given NO notebookIndex.index() was called + //when + List> result = notebookIndex.query("anything"); + //then + assertThat(result).isEmpty(); + //assert logs were printed + //"ERROR org.apache.zeppelin.search.SearchService:97 - Failed to open index dir RAMDirectory" + } + + @Test public void canIndexAndReIndex() throws IOException { + //given + Note note1 = newNoteWithParapgraph("Notebook1", "test"); + Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); + + //when + Paragraph p2 = note2.getLastParagraph(); + p2.setText("test indeed"); + notebookIndex.updateIndexDoc(note2); + + //then + List> results = notebookIndex.query("all"); + assertThat(results).isEmpty(); + + results = notebookIndex.query("indeed"); + assertThat(results).isNotEmpty(); + } + + @Test public void canDeleteNull() throws IOException { + //give + // looks like a bug in web UI: it tries to delete a note twice (after it has just been deleted) + //when + notebookIndex.deleteIndexDocs(null); + } + + @Test public void canDeleteFromIndex() throws IOException { + //given + Note note1 = newNoteWithParapgraph("Notebook1", "test"); + Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); + assertThat(resultForQuery("Notebook2")).isNotEmpty(); + + //when + notebookIndex.deleteIndexDocs(note2); + + //then + assertThat(notebookIndex.query("all")).isEmpty(); + assertThat(resultForQuery("Notebook2")).isEmpty(); + + List> results = resultForQuery("test"); + assertThat(results).isNotEmpty(); + assertThat(results.size()).isEqualTo(1); + } + + @Test public void indexParagraphUpdatedOnNoteSave() throws IOException { + //given: total 2 notebooks, 3 paragraphs + Note note1 = newNoteWithParapgraph("Notebook1", "test"); + Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); + assertThat(resultForQuery("test").size()).isEqualTo(3); + + //when + Paragraph p1 = note1.getLastParagraph(); + p1.setText("no no no"); + note1.persist(); + + //then + assertThat(resultForQuery("Notebook1").size()).isEqualTo(1); + + List> results = resultForQuery("test"); + assertThat(results).isNotEmpty(); + assertThat(results.size()).isEqualTo(2); + + //does not include Notebook1's paragraph any more + for (Map result: results) { + assertThat(result.get("id").startsWith(note1.getId())).isFalse();; + } + } + + @Test public void indexNoteNameUpdatedOnNoteSave() throws IOException { + //given: total 2 notebooks, 3 paragraphs + Note note1 = newNoteWithParapgraph("Notebook1", "test"); + Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); + assertThat(resultForQuery("test").size()).isEqualTo(3); + + //when + note1.setName("NotebookN"); + note1.persist(); + + //then + assertThat(resultForQuery("Notebook1")).isEmpty(); + assertThat(resultForQuery("NotebookN")).isNotEmpty(); + assertThat(resultForQuery("NotebookN").size()).isEqualTo(1); + } + + private List> resultForQuery(String q) { + return notebookIndex.query(q); + } + + /** + * Creates a new Note \w given name, + * adds a new paragraph \w given text + * + * @param noteName name of the note + * @param parText text of the paragraph + * @return Note + */ + private Note newNoteWithParapgraph(String noteName, String parText) { + Note note1 = newNote(noteName); + addParagraphWithText(note1, parText); + return note1; + } + + /** + * Creates a new Note \w given name, + * adds N paragraphs \w given texts + */ + private Note newNoteWithParapgraphs(String noteName, String... parTexts) { + Note note1 = newNote(noteName); + for (String parText : parTexts) { + addParagraphWithText(note1, parText); + } + return note1; + } + + private Paragraph addParagraphWithText(Note note, String text) { + Paragraph p = note.addParagraph(); + p.setText(text); + return p; + } + + private Note newNote(String name) { + Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex); + note.setName(name); + return note; + } + +}