From ac546b24cc5ff1d52923b9e1e6e61050137328c9 Mon Sep 17 00:00:00 2001 From: David Tzoor Date: Sat, 18 Jun 2016 14:47:28 +0300 Subject: [PATCH 01/88] add force layout button --- .../src/app/notebook/paragraph/paragraph-chart-selector.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html index 76135b1d5c3..a6b81d11fd7 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html @@ -40,6 +40,10 @@ ng-class="{'active': isGraphMode('scatterChart')}" ng-click="setGraphMode('scatterChart', true)"> + - - + diff --git a/zeppelin-web/src/app/notebook/notebook.css b/zeppelin-web/src/app/notebook/notebook.css index 6c9a190d552..c86eb9db986 100644 --- a/zeppelin-web/src/app/notebook/notebook.css +++ b/zeppelin-web/src/app/notebook/notebook.css @@ -223,6 +223,13 @@ padding-left: 5px; } +.setting-btn { + position: relative; + top: 2px; + margin-right: 4px; + cursor: pointer; +} + .cron-preset-container { padding: 10px 20px 0 20px; font-weight: normal; diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index e2f40c86b68..cb1d91cb898 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -63,7 +63,6 @@ angular.module('zeppelinWebApp') }, 500); }; - var vm = this; vm.notes = notebookListDataFactory; vm.connected = websocketMsgSrv.isConnected(); @@ -71,13 +70,6 @@ angular.module('zeppelinWebApp') vm.arrayOrderingSrv = arrayOrderingSrv; $scope.searchForm = searchService; - if ($rootScope.ticket) { - $rootScope.fullUsername = $rootScope.ticket.principal; - $rootScope.truncatedUsername = $rootScope.ticket.principal; - } - - var MAX_USERNAME_LENGTH=16; - angular.element('#notebook-list').perfectScrollbar({suppressScrollX: true}); $scope.$on('setNoteMenu', function(event, notes) { @@ -88,22 +80,7 @@ angular.module('zeppelinWebApp') vm.connected = param; }); - $scope.checkUsername = function () { - if ($rootScope.ticket) { - if ($rootScope.ticket.principal.length <= MAX_USERNAME_LENGTH) { - $rootScope.truncatedUsername = $rootScope.ticket.principal; - } - else { - $rootScope.truncatedUsername = $rootScope.ticket.principal.substr(0, MAX_USERNAME_LENGTH) + '..'; - } - } - if (_.isEmpty($rootScope.truncatedUsername)) { - $rootScope.truncatedUsername = 'Connected'; - } - }; - $scope.$on('loginSuccess', function(event, param) { - $scope.checkUsername(); loadNotes(); }); @@ -146,7 +123,6 @@ angular.module('zeppelinWebApp') }; function getZeppelinVersion() { - console.log('version'); $http.get(baseUrlSrv.getRestApiBase() + '/version').success( function(data, status, headers, config) { $rootScope.zeppelinVersion = data.body; @@ -161,6 +137,5 @@ angular.module('zeppelinWebApp') getZeppelinVersion(); vm.loadNotes(); - $scope.checkUsername(); }); diff --git a/zeppelin-web/src/components/navbar/navbar.html b/zeppelin-web/src/components/navbar/navbar.html index 5e7de3bf61f..9ba2cb14d95 100644 --- a/zeppelin-web/src/components/navbar/navbar.html +++ b/zeppelin-web/src/components/navbar/navbar.html @@ -44,24 +44,15 @@
  • -
  • +
  • -
  • - Interpreter -
    • -
  • - Credential -
    • -
  • - Configuration -
    • - From 6497008cc309f88bd8816c3c3e374887db524aae Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 23 Jun 2016 23:52:29 -0700 Subject: [PATCH 36/88] [ZEPPELIN-936] Fix flaky test SparkRTest ### What is this PR for? This PR fixes test failure described in ZEPPELIN-936, or hanging on SparkRTest ``` Spark version detected 1.6.1 23:52:30,005 INFO org.apache.zeppelin.notebook.Paragraph:252 - run paragraph 20160623-235230_1368989448 using r org.apache.zeppelin.interpreter.LazyOpenInterpreter5221ff81 No output has been received in the last 10 minutes, this potentially indicates a stalled build or something wrong with the build itself. The build has been terminated ``` ### What type of PR is it? Hot Fix ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-936 Author: Lee moon soo Closes #1078 from Leemoonsoo/disable_flaky_test and squashes the following commits: 4872d66 [Lee moon soo] trigger ci b5b7b83 [Lee moon soo] trigger ci 4f75ade [Lee moon soo] trigger ci 31a247c [Lee moon soo] trigger ci 9183e2a [Lee moon soo] trigger ci 582b6e1 [Lee moon soo] reduce spark.cores.max on testing --- .../java/org/apache/zeppelin/rest/AbstractTestRestApi.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 5ea3c0943a5..7bedd284c59 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -127,8 +127,9 @@ protected static void startUp() throws Exception { } } - // set spark master + // set spark master and other properties sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071"); + sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2"); // set spark home for pyspark sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome()); From 8154c87219262248532fe15ea97cc84817f7b862 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Thu, 23 Jun 2016 19:06:31 +0530 Subject: [PATCH 37/88] [ZEPPELIN-1052] Application does not logout user when authcBasic is used ### What is this PR for? This PR is WRT to [this](http://apache-zeppelin-users-incubating-mailing-list.75479.x6.nabble.com/Fwd-Authentication-in-zeppelin-td3354.html) mail thread (Authentication in zeppelin) Where in if authcBasic mechanisim is used then on clicking logout, the user doesn't gets logout. ### What type of PR is it? [Bug Fix] ### Todos * [x] - set username and password false on logout ### What is the Jira issue? * [ZEPPELIN-533](https://issues.apache.org/jira/browse/ZEPPELIN-1052) ### How should this be tested? In shiro.ini conf set `/** = authcBasic`, then start the zeppelin server. - try login as admin/password1 - now try to logout (this should work) ### Questions: * Does the licenses files need update? n/a * Is there breaking changes for older versions? n/a * Does this needs documentation? n/a Author: Prabhjyot Singh Closes #1071 from prabhjyotsingh/ZEPPELIN-1052 and squashes the following commits: 6f4dd09 [Prabhjyot Singh] force authcBasic by setting credentials as false:false b3d6935 [Prabhjyot Singh] set username and password false on logout --- .../components/navbar/navbar.controller.js | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index cb1d91cb898..31e595e3714 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -30,7 +30,7 @@ angular.module('zeppelinWebApp') return notebook; } - if (notebook.children) { + if (notebook.children) { filteringNote(notebook.children, filteredNotes); } }); @@ -85,23 +85,31 @@ angular.module('zeppelinWebApp') }); $scope.logout = function() { - $http.post(baseUrlSrv.getRestApiBase()+'/login/logout') - .success(function(data, status, headers, config) { - $rootScope.userName = ''; - $rootScope.ticket.principal = ''; - $rootScope.ticket.ticket = ''; - $rootScope.ticket.roles = ''; - BootstrapDialog.show({ - message: 'Logout Success' - }); - setTimeout(function() { - window.location = '#'; - window.location.reload(); - }, 1000); - }). - error(function(data, status, headers, config) { - console.log('Error %o %o', status, data.message); - }); + var logoutURL = baseUrlSrv.getRestApiBase() + '/login/logout'; + var request = new XMLHttpRequest(); + + //force authcBasic (if configured) to logout by setting credentials as false:false + request.open('post', logoutURL, true, 'false', 'false'); + request.onreadystatechange = function() { + if (request.readyState === 4) { + if (request.status === 401 || request.status === 405) { + $rootScope.userName = ''; + $rootScope.ticket.principal = ''; + $rootScope.ticket.ticket = ''; + $rootScope.ticket.roles = ''; + BootstrapDialog.show({ + message: 'Logout Success' + }); + setTimeout(function() { + window.location.replace('/'); + }, 1000); + } else { + request.open('post', logoutURL, true, 'false', 'false'); + request.send(); + } + } + }; + request.send(); }; $scope.search = function(searchTerm) { From 878a8c76cfc754974f35fbb5e74ebe3821152d7f Mon Sep 17 00:00:00 2001 From: sadikovi Date: Tue, 21 Jun 2016 18:00:26 +1200 Subject: [PATCH 38/88] [ZEPPELIN-1034] Add Spark Interpreter option to not import implicits ### What is this PR for? This PR adds option `zeppelin.spark.importImplicit` to not import `SQLContext` implicits, UDF collections. It is `true`, which means importing implicit functions and has been the default behaviour. ### What type of PR is it? Feature ### Todos ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-1034 ### How should this be tested? Added unit-tests, also manual testing using similar to unit-tests scenario. ### Screenshots (if appropriate) ### Questions: * Does this needs documentation? Documentation is updated to include description for new option. Author: sadikovi Closes #1049 from sadikovi/ZEPPELIN-1034 and squashes the following commits: 772ea78 [sadikovi] update tests 4aabdef [sadikovi] add option to import implicits --- conf/zeppelin-env.cmd.template | 3 +- conf/zeppelin-env.sh.template | 5 +-- docs/interpreter/spark.md | 7 +++- .../zeppelin/spark/SparkInterpreter.java | 20 ++++++++---- .../main/resources/interpreter-setting.json | 6 ++++ .../zeppelin/spark/SparkInterpreterTest.java | 32 +++++++++++++++++++ 6 files changed, 62 insertions(+), 11 deletions(-) diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template index 59953dcc38c..d85e59f2709 100644 --- a/conf/zeppelin-env.cmd.template +++ b/conf/zeppelin-env.cmd.template @@ -55,12 +55,13 @@ REM set HADOOP_CONF_DIR REM yarn-site.xml is located in configuration REM Pyspark (supported with Spark 1.2.1 and above) REM To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI REM set PYSPARK_PYTHON REM path to the python command. must be the same path on the driver(Zeppelin) and all workers. -REM set PYTHONPATH +REM set PYTHONPATH REM Spark interpreter options REM REM set ZEPPELIN_SPARK_USEHIVECONTEXT REM Use HiveContext instead of SQLContext if set true. true by default. REM set ZEPPELIN_SPARK_CONCURRENTSQL REM Execute multiple SQL concurrently if set true. false by default. +REM set ZEPPELIN_SPARK_IMPORTIMPLICIT REM Import implicits, UDF collection, and sql if set true. true by default. REM set ZEPPELIN_SPARK_MAXRESULT REM Max number of SparkSQL result to display. 1000 by default. REM ZeppelinHub connection configuration diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index be6f3dd83f1..52e36f7b5f6 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -55,15 +55,17 @@ # Pyspark (supported with Spark 1.2.1 and above) # To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI # export PYSPARK_PYTHON # path to the python command. must be the same path on the driver(Zeppelin) and all workers. -# export PYTHONPATH +# export PYTHONPATH ## Spark interpreter options ## ## # export ZEPPELIN_SPARK_USEHIVECONTEXT # Use HiveContext instead of SQLContext if set true. true by default. # export ZEPPELIN_SPARK_CONCURRENTSQL # Execute multiple SQL concurrently if set true. false by default. +# export ZEPPELIN_SPARK_IMPORTIMPLICIT # Import implicits, UDF collection, and sql if set true. true by default. # export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default. # export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000 + #### HBase interpreter configuration #### ## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set @@ -75,4 +77,3 @@ # export ZEPPELINHUB_API_ADDRESS # Refers to the address of the ZeppelinHub service in use # export ZEPPELINHUB_API_TOKEN # Refers to the Zeppelin instance token of the user # export ZEPPELINHUB_USER_KEY # Optional, when using Zeppelin with authentication. - diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index 30da219ac03..df5e83176f2 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -45,7 +45,7 @@ Spark Interpreter group, which consists of five interpreters. ## Configuration -The Spark interpreter can be configured with properties provided by Zeppelin. +The Spark interpreter can be configured with properties provided by Zeppelin. You can also set other Spark properties which are not listed in the table. For a list of additional properties, refer to [Spark Available Properties](http://spark.apache.org/docs/latest/configuration.html#available-properties). @@ -111,6 +111,11 @@ You can also set other Spark properties which are not listed in the table. For a + + + + +
    true Use HiveContext instead of SQLContext if it is true.
    zeppelin.spark.importImplicittrueImport implicits, UDF collection, and sql if set true.
    Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps. diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 0bbe418a2cb..6783378efda 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -176,6 +176,10 @@ private boolean useHiveContext() { return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.useHiveContext")); } + private boolean importImplicit() { + return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.importImplicit")); + } + public SQLContext getSQLContext() { synchronized (sharedInterpreterLock) { if (sqlc == null) { @@ -250,7 +254,7 @@ public SparkContext createSparkContext() { | IllegalArgumentException | InvocationTargetException e) { // continue instead of: throw new InterpreterException(e); // Newer Spark versions (like the patched CDH5.7.0 one) don't contain this method - logger.warn(String.format("Spark method classServerUri not available due to: [%s]", + logger.warn(String.format("Spark method classServerUri not available due to: [%s]", e.getMessage())); } } @@ -540,12 +544,14 @@ public void open() { + "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]"); intp.interpret("import org.apache.spark.SparkContext._"); - if (sparkVersion.oldSqlContextImplicits()) { - intp.interpret("import sqlContext._"); - } else { - intp.interpret("import sqlContext.implicits._"); - intp.interpret("import sqlContext.sql"); - intp.interpret("import org.apache.spark.sql.functions._"); + if (importImplicit()) { + if (sparkVersion.oldSqlContextImplicits()) { + intp.interpret("import sqlContext._"); + } else { + intp.interpret("import sqlContext.implicits._"); + intp.interpret("import sqlContext.sql"); + intp.interpret("import org.apache.spark.sql.functions._"); + } } } diff --git a/spark/src/main/resources/interpreter-setting.json b/spark/src/main/resources/interpreter-setting.json index 1d36a2949c2..d46801ba634 100644 --- a/spark/src/main/resources/interpreter-setting.json +++ b/spark/src/main/resources/interpreter-setting.json @@ -77,6 +77,12 @@ "propertyName": "zeppelin.spark.maxResult", "defaultValue": "1000", "description": "Max number of SparkSQL result to display." + }, + "zeppelin.spark.importImplicit": { + "envName": "ZEPPELIN_SPARK_IMPORTIMPLICIT", + "propertyName": "zeppelin.spark.importImplicit", + "defaultValue": "true", + "description": "Import implicits, UDF collection, and sql if set true. true by default." } } }, diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index 409f938d63d..eb8d876021a 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -69,6 +69,7 @@ public static Properties getSparkTestProperties() { p.setProperty("spark.app.name", "Zeppelin Test"); p.setProperty("zeppelin.spark.useHiveContext", "true"); p.setProperty("zeppelin.spark.maxResult", "1000"); + p.setProperty("zeppelin.spark.importImplicit", "true"); return p; } @@ -228,4 +229,35 @@ public void shareSingleSparkContext() throws InterruptedException { repl2.close(); } + + @Test + public void testEnableImplicitImport() { + // Set option of importing implicits to "true", and initialize new Spark repl + Properties p = getSparkTestProperties(); + p.setProperty("zeppelin.spark.importImplicit", "true"); + SparkInterpreter repl2 = new SparkInterpreter(p); + repl2.setInterpreterGroup(intpGroup); + intpGroup.get("note").add(repl2); + + repl2.open(); + String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")"; + assertEquals(Code.SUCCESS, repl2.interpret(ddl, context).code()); + repl2.close(); + } + + @Test + public void testDisableImplicitImport() { + // Set option of importing implicits to "false", and initialize new Spark repl + // this test should return error status when creating DataFrame from sequence + Properties p = getSparkTestProperties(); + p.setProperty("zeppelin.spark.importImplicit", "false"); + SparkInterpreter repl2 = new SparkInterpreter(p); + repl2.setInterpreterGroup(intpGroup); + intpGroup.get("note").add(repl2); + + repl2.open(); + String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")"; + assertEquals(Code.ERROR, repl2.interpret(ddl, context).code()); + repl2.close(); + } } From 57e0dc88310b9dbdece780beab5c9e3e496be1b9 Mon Sep 17 00:00:00 2001 From: AhyoungRyu Date: Wed, 22 Jun 2016 15:06:35 -0700 Subject: [PATCH 39/88] [ZEPPELIN-1028] Fix exported notebook importing error ### What is this PR for? This bug seems to be produced by #862. Currently a exported notebook is not imported with below error message. ``` ERROR [2016-06-20 17:19:21,797] ({qtp559670971-14} NotebookServer.java[onMessage]:231) - Can't handle message com.google.gson.JsonSyntaxException: 2016-06-20T14:33:31-0700 at com.google.gson.internal.bind.DateTypeAdapter.deserializeToDate(DateTypeAdapter.java:81) at com.google.gson.internal.bind.DateTypeAdapter.read(DateTypeAdapter.java:66) at com.google.gson.internal.bind.DateTypeAdapter.read(DateTypeAdapter.java:41) at com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$1.read(ReflectiveTypeAdapterFactory.java:93) at com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$Adapter.read(ReflectiveTypeAdapterFactory.java:172) at com.google.gson.internal.bind.TypeAdapterRuntimeTypeWrapper.read(TypeAdapterRuntimeTypeWrapper.java:40) at com.google.gson.internal.bind.CollectionTypeAdapterFactory$Adapter.read(CollectionTypeAdapterFactory.java:81) at com.google.gson.internal.bind.CollectionTypeAdapterFactory$Adapter.read(CollectionTypeAdapterFactory.java:60) at com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$1.read(ReflectiveTypeAdapterFactory.java:93) at com.google.gson.internal.bind.ReflectiveTypeAdapterFactory$Adapter.read(ReflectiveTypeAdapterFactory.java:172) at com.google.gson.Gson.fromJson(Gson.java:791) at org.apache.zeppelin.notebook.Notebook.importNote(Notebook.java:199) at org.apache.zeppelin.socket.NotebookServer.importNote(NotebookServer.java:656) at org.apache.zeppelin.socket.NotebookServer.onMessage(NotebookServer.java:175) at org.apache.zeppelin.socket.NotebookSocket.onWebSocketText(NotebookSocket.java:56) at org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextMessage(JettyListenerEventDriver.java:128) at org.eclipse.jetty.websocket.common.message.SimpleTextMessage.messageComplete(SimpleTextMessage.java:69) at org.eclipse.jetty.websocket.common.events.AbstractEventDriver.appendMessage(AbstractEventDriver.java:65) at org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextFrame(JettyListenerEventDriver.java:122) at org.eclipse.jetty.websocket.common.events.AbstractEventDriver.incomingFrame(AbstractEventDriver.java:161) at org.eclipse.jetty.websocket.common.WebSocketSession.incomingFrame(WebSocketSession.java:309) at org.eclipse.jetty.websocket.common.extensions.ExtensionStack.incomingFrame(ExtensionStack.java:214) at org.eclipse.jetty.websocket.common.Parser.notifyFrame(Parser.java:220) at org.eclipse.jetty.websocket.common.Parser.parse(Parser.java:258) at org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.readParse(AbstractWebSocketConnection.java:632) at org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.onFillable(AbstractWebSocketConnection.java:480) at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:745) Caused by: java.text.ParseException: Unparseable date: "2016-06-20T14:33:31-0700" at java.text.DateFormat.parse(DateFormat.java:366) at com.google.gson.internal.bind.DateTypeAdapter.deserializeToDate(DateTypeAdapter.java:79) ``` ### What type of PR is it? Bug Fix ### Todos ### What is the Jira issue? [ZEPPELIN-1028](https://issues.apache.org/jira/browse/ZEPPELIN-1028) ### How should this be tested? 1. Apply this patch (Build the source and restart Zeppelin) 2. Export a notebook and try to import it again 3. It should be imported as before ### Screenshots (if appropriate) With this patch, we can import the below two types of date format notebooks. - Exported after #862 merged : `yyyy-MM-dd'T'HH:mm:ssZ` screen shot 2016-06-22 at 12 18 01 pm - Exported before #862 merged : `MMM dd, yyyy HH:mm:ss` screen shot 2016-06-22 at 12 17 31 pm ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: AhyoungRyu Closes #1055 from AhyoungRyu/ZEPPELIN-1028 and squashes the following commits: 8f91d2d [AhyoungRyu] Fix class description 829bcae [AhyoungRyu] Rename NotebookImportSerializer -> NotebookImportDeserializer 2dff9bb [AhyoungRyu] Support two date format for backward compatibility 2d8fc66 [AhyoungRyu] Remove new line 7c493bf [AhyoungRyu] Change date format in importNote 479c0d0 [AhyoungRyu] Fix exported notebook importing error --- .../zeppelin/socket/NotebookServer.java | 3 +- .../apache/zeppelin/notebook/Notebook.java | 4 +- .../notebook/NotebookImportDeserializer.java | 53 +++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 17650dd10cc..f4cf9d9efb1 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -77,8 +77,7 @@ String getKey() { } private static final Logger LOG = LoggerFactory.getLogger(NotebookServer.class); - Gson gson = new GsonBuilder() - .setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); + Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); final Map> noteSocketMap = new HashMap<>(); final Queue connectedSockets = new ConcurrentLinkedQueue<>(); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index adaaa2ac4d0..d590223b2e0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -195,7 +195,9 @@ public Note importNote(String sourceJson, String noteName, AuthenticationInfo su throws IOException { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); - Gson gson = gsonBuilder.create(); + + Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()) + .create(); JsonReader reader = new JsonReader(new StringReader(sourceJson)); reader.setLenient(true); Note newNote; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java new file mode 100644 index 00000000000..1aadf75e67f --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; + +import java.lang.reflect.Type; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.Locale; + +/** + * importNote date format deserializer + */ +public class NotebookImportDeserializer implements JsonDeserializer { + private static final String[] DATE_FORMATS = new String[] { + "yyyy-MM-dd'T'HH:mm:ssZ", + "MMM dd, yyyy HH:mm:ss" + }; + + @Override + public Date deserialize(JsonElement jsonElement, Type typeOF, + JsonDeserializationContext context) throws JsonParseException { + for (String format : DATE_FORMATS) { + try { + return new SimpleDateFormat(format, Locale.US).parse(jsonElement.getAsString()); + } catch (ParseException e) { + } + } + throw new JsonParseException("Unparsable date: \"" + jsonElement.getAsString() + + "\". Supported formats: " + Arrays.toString(DATE_FORMATS)); + } +} From f55290f49449ca6db0f06985ee3ecbce5864a05e Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Fri, 24 Jun 2016 03:18:11 +0900 Subject: [PATCH 40/88] [MINOR] Remove unused util methods and its tests ### What is this PR for? removing unused codes and its test ### What type of PR is it? [Refactoring] ### Todos * [x] - Remove codes ### What is the Jira issue? N/A ### How should this be tested? ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jongyoul Lee Closes #1072 from jongyoul/minor-remove-unused-codes and squashes the following commits: d89b0b1 [Jongyoul Lee] Removed duplicated setting of dependency d624cb5 [Jongyoul Lee] Removed unused codes 06d44d0 [Jongyoul Lee] Remove unused util methods and its tests --- zeppelin-server/pom.xml | 7 - .../java/org/apache/zeppelin/util/Util.java | 161 ------------------ .../org/apache/zeppelin/util/UtilTest.java | 106 ------------ .../apache/zeppelin/util/UtilsForTests.java | 124 -------------- 4 files changed, 398 deletions(-) delete mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilTest.java delete mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilsForTests.java diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 270eb5839d2..b88e1757fcf 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -256,13 +256,6 @@ test - - org.apache.httpcomponents - httpclient - 4.3.6 - test - - org.scalatest scalatest_2.10 diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java index f9dec0fd48c..e8c9076115a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java @@ -18,13 +18,8 @@ package org.apache.zeppelin.util; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; import java.util.Properties; /** @@ -44,162 +39,6 @@ public class Util { } } - public static String[] split(String str, char split) { - return split(str, new String[] {String.valueOf(split)}, false); - } - - public static String[] split(String str, String[] splitters, boolean includeSplitter) { - String escapeSeq = "\"',;<%>"; - char escapeChar = '\\'; - String[] blockStart = new String[] {"\"", "'", "<%", "N_<"}; - String[] blockEnd = new String[] {"\"", "'", "%>", "N_>"}; - - return split(str, escapeSeq, escapeChar, blockStart, blockEnd, splitters, includeSplitter); - - } - - public static String[] split(String str, String escapeSeq, char escapeChar, String[] blockStart, - String[] blockEnd, String[] splitters, boolean includeSplitter) { - - List splits = new ArrayList(); - - String curString = ""; - - boolean escape = false; // true when escape char is found - int lastEscapeOffset = -1; - int blockStartPos = -1; - List blockStack = new LinkedList(); - - for (int i = 0; i < str.length(); i++) { - char c = str.charAt(i); - - // escape char detected - if (c == escapeChar && escape == false) { - escape = true; - continue; - } - - // escaped char comes - if (escape == true) { - if (escapeSeq.indexOf(c) < 0) { - curString += escapeChar; - } - curString += c; - escape = false; - lastEscapeOffset = curString.length(); - continue; - } - - if (blockStack.size() > 0) { // inside of block - curString += c; - // check multichar block - boolean multicharBlockDetected = false; - for (int b = 0; b < blockStart.length; b++) { - if (blockStartPos >= 0 - && getBlockStr(blockStart[b]).compareTo(str.substring(blockStartPos, i)) == 0) { - blockStack.remove(0); - blockStack.add(0, b); - multicharBlockDetected = true; - break; - } - } - if (multicharBlockDetected == true) { - continue; - } - - // check if current block is nestable - if (isNestedBlock(blockStart[blockStack.get(0)]) == true) { - // try to find nested block start - - if (curString.substring(lastEscapeOffset + 1).endsWith( - getBlockStr(blockStart[blockStack.get(0)])) == true) { - blockStack.add(0, blockStack.get(0)); // block is started - blockStartPos = i; - continue; - } - } - - // check if block is finishing - if (curString.substring(lastEscapeOffset + 1).endsWith( - getBlockStr(blockEnd[blockStack.get(0)]))) { - // the block closer is one of the splitters (and not nested block) - if (isNestedBlock(blockEnd[blockStack.get(0)]) == false) { - for (String splitter : splitters) { - if (splitter.compareTo(getBlockStr(blockEnd[blockStack.get(0)])) == 0) { - splits.add(curString); - if (includeSplitter == true) { - splits.add(splitter); - } - curString = ""; - lastEscapeOffset = -1; - - break; - } - } - } - blockStartPos = -1; - blockStack.remove(0); - continue; - } - - } else { // not in the block - boolean splitted = false; - for (String splitter : splitters) { - // forward check for splitter - if (splitter.compareTo( - str.substring(i, Math.min(i + splitter.length(), str.length()))) == 0) { - splits.add(curString); - if (includeSplitter == true) { - splits.add(splitter); - } - curString = ""; - lastEscapeOffset = -1; - i += splitter.length() - 1; - splitted = true; - break; - } - } - if (splitted == true) { - continue; - } - - // add char to current string - curString += c; - - // check if block is started - for (int b = 0; b < blockStart.length; b++) { - if (curString.substring(lastEscapeOffset + 1) - .endsWith(getBlockStr(blockStart[b])) == true) { - blockStack.add(0, b); // block is started - blockStartPos = i; - break; - } - } - } - } - if (curString.length() > 0) { - splits.add(curString.trim()); - } - return splits.toArray(new String[] {}); - - } - - private static String getBlockStr(String blockDef) { - if (blockDef.startsWith("N_")) { - return blockDef.substring("N_".length()); - } else { - return blockDef; - } - } - - private static boolean isNestedBlock(String blockDef) { - if (blockDef.startsWith("N_")) { - return true; - } else { - return false; - } - } - /** * Get Zeppelin version * diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilTest.java deleted file mode 100644 index 713166d7970..00000000000 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.util; - -import org.apache.zeppelin.util.Util; - -import junit.framework.TestCase; - -public class UtilTest extends TestCase { - - @Override - protected void setUp() throws Exception { - super.setUp(); - } - - @Override - protected void tearDown() throws Exception { - super.tearDown(); - } - - public void testSplitIncludingToken() { - String[] token = Util.split("hello | \"world '>|hehe\" > next >> sink", new String[]{"|", ">>", ">"}, true); - assertEquals(7, token.length); - assertEquals(" \"world '>|hehe\" ", token[2]); - } - - public void testSplitExcludingToken() { - String[] token = Util.split("hello | \"world '>|hehe\" > next >> sink", new String[]{"|", ">>", ">"}, false); - assertEquals(4, token.length); - assertEquals(" \"world '>|hehe\" ", token[1]); - } - - public void testSplitWithSemicolonEnd(){ - String[] token = Util.split("show tables;", ';'); - assertEquals(1, token.length); - assertEquals("show tables", token[0]); - } - - public void testEscapeTemplate(){ - String[] token = Util.split("select * from <%=table%> limit 1 > output", '>'); - assertEquals(2, token.length); - assertEquals("output", token[1]); - } - - public void testSplit(){ - String [] op = new String[]{";", "|", ">>", ">"}; - - String str = "CREATE external table news20b_train (\n"+ - " rowid int,\n"+ - " label int,\n"+ - " features ARRAY\n"+ - ")\n"+ - "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' \n"+ - "COLLECTION ITEMS TERMINATED BY \",\" \n"+ - "STORED AS TEXTFILE;\n"; - Util.split(str, op, true); - - } - - public void testSplitDifferentBlockStartEnd(){ - String [] op = new String[]{";", "|", ">>", ">"}; - String escapeSeq = "\"',;<%>!"; - char escapeChar = '\\'; - String [] blockStart = new String[]{ "\"", "'", "<%", "<", "!"}; - String [] blockEnd = new String[]{ "\"", "'", "%>", ">", ";" }; - String [] t = Util.split("!echo a;!echo b;", escapeSeq, escapeChar, blockStart, blockEnd, op, true); - assertEquals(4, t.length); - assertEquals("!echo a;", t[0]); - assertEquals(";", t[1]); - assertEquals("!echo b;", t[2]); - assertEquals(";", t[3]); - } - - public void testNestedBlock(){ - String [] op = new String[]{";", "|", ">>", ">"}; - String escapeSeq = "\"',;<%>!"; - char escapeChar = '\\'; - String [] blockStart = new String[]{ "\"", "'", "<%", "N_<", "<", "!"}; - String [] blockEnd = new String[]{ "\"", "'", "%>", "N_>", ";", ";" }; - String [] t = Util.split("array > tags|aa", escapeSeq, escapeChar, blockStart, blockEnd, op, true); - assertEquals(3, t.length); - assertEquals("array > tags", t[0]); - assertEquals("aa", t[2]); - } - - public void testGetVersion(){ - String version = Util.getVersion(); - assertNotNull(version); - System.out.println(version); - } -} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilsForTests.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilsForTests.java deleted file mode 100644 index 22002ee452a..00000000000 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilsForTests.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.util; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -public class UtilsForTests { - - static Logger LOGGER = LoggerFactory.getLogger(UtilsForTests.class); - - public static File createTmpDir() throws Exception { - File tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); - tmpDir.mkdir(); - return tmpDir; - - } - /* - private static final String HADOOP_DIST="http://apache.mirror.cdnetworks.com/hadoop/common/hadoop-1.2.1/hadoop-1.2.1-bin.tar.gz"; - //private static final String HADOOP_DIST="http://www.us.apache.org/dist/hadoop/common/hadoop-1.2.1/hadoop-1.2.1-bin.tar.gz"; - - public static void getHadoop() throws MalformedURLException, IOException{ - setEnv("HADOOP_HOME", new File("./target/hadoop-1.2.1").getAbsolutePath()); - if(new File("./target/hadoop-1.2.1").isDirectory()) return; - //System.out.println("Downloading a hadoop distribution ... it will take a while"); - //FileUtils.copyURLToFile(new URL(HADOOP_DIST), new File("/tmp/zp_test_hadoop-bin.tar.gz")); - System.out.println("Unarchive hadoop distribution ... "); - new File("./target").mkdir(); - Runtime.getRuntime().exec("tar -xzf /tmp/zp_test_hadoop-bin.tar.gz -C ./target"); - } - */ - - public static void delete(File file) { - if (file.isFile()) file.delete(); - else if (file.isDirectory()) { - File[] files = file.listFiles(); - if (files != null && files.length > 0) { - for (File f : files) { - delete(f); - } - } - file.delete(); - } - } - - /** - * Utility method to create a file (if does not exist) and populate it the the given content - * - * @param path to file - * @param content of the file - * @throws IOException - */ - public static void createFileWithContent(String path, String content) throws IOException { - File f = new File(path); - if (!f.exists()) { - stringToFile(content, f); - } - } - - public static void stringToFile(String string, File file) throws IOException { - FileOutputStream out = new FileOutputStream(file); - out.write(string.getBytes()); - out.close(); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public static void setEnv(String k, String v) { - Map newenv = new HashMap(); - newenv.put(k, v); - try { - Class processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment"); - Field theEnvironmentField = processEnvironmentClass.getDeclaredField("theEnvironment"); - theEnvironmentField.setAccessible(true); - Map env = (Map) theEnvironmentField.get(null); - env.putAll(newenv); - Field theCaseInsensitiveEnvironmentField = processEnvironmentClass.getDeclaredField("theCaseInsensitiveEnvironment"); - theCaseInsensitiveEnvironmentField.setAccessible(true); - Map cienv = (Map) theCaseInsensitiveEnvironmentField.get(null); - cienv.putAll(newenv); - } catch (NoSuchFieldException e) { - try { - Class[] classes = Collections.class.getDeclaredClasses(); - Map env = System.getenv(); - for (Class cl : classes) { - if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { - Field field = cl.getDeclaredField("m"); - field.setAccessible(true); - Object obj = field.get(env); - Map map = (Map) obj; - map.clear(); - map.putAll(newenv); - } - } - } catch (Exception e2) { - LOGGER.error(e2.toString(), e2); - } - } catch (Exception e1) { - LOGGER.error(e1.toString(), e1); - } - } -} From 510cecd3896f1d9b0551f7f2d5ecd5f3db655427 Mon Sep 17 00:00:00 2001 From: Prabhjyot Singh Date: Sat, 25 Jun 2016 08:31:50 +0530 Subject: [PATCH 41/88] [HOTFIX] Fix flaky AuthenticationIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What is this PR for? This is hotfix for CI failing for Tests in error: AuthenticationIT.testGroupPermission:177->authenticationUser:109->AbstractZeppelinIT.pollingWait:114 » Timeout ### What type of PR is it? [Hot Fix] ### Todos * [x] - Add check for request.status === 500, while logout ### What is the Jira issue? - [ZEPPELIN-1009](https://issues.apache.org/jira/browse/ZEPPELIN-1009) ### How should this be tested? Check for CI green ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? n/a * Is there breaking changes for older versions? n/a * Does this needs documentation? n/a Author: Prabhjyot Singh Closes #1084 from prabhjyotsingh/hotfix/testGroupPermission and squashes the following commits: 50b479b [Prabhjyot Singh] check for request.status === 500 --- zeppelin-web/src/components/navbar/navbar.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 31e595e3714..16209344f56 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -92,7 +92,7 @@ angular.module('zeppelinWebApp') request.open('post', logoutURL, true, 'false', 'false'); request.onreadystatechange = function() { if (request.readyState === 4) { - if (request.status === 401 || request.status === 405) { + if (request.status === 401 || request.status === 405 || request.status === 500) { $rootScope.userName = ''; $rootScope.ticket.principal = ''; $rootScope.ticket.ticket = ''; From 056eee8342699f93e01944c66ba0b053be4c5382 Mon Sep 17 00:00:00 2001 From: Sagar Kulkarni Date: Fri, 24 Jun 2016 12:37:47 +0530 Subject: [PATCH 42/88] [ZEPPELIN-961] Longer names to the notebook make the name-text and buttons float outside the border. ### What is this PR for? To correct the styling of notebook header which has notebook name, in case when the name is longer. Feature : We should not limit the name size, but, while showing we can limit the name text to be shown. ### What type of PR is it? [Bug Fix] ### Todos ### What is the Jira issue? [ZEPPELIN-961](https://issues.apache.org/jira/browse/ZEPPELIN-961) ### How should this be tested? - Open any notebook. - Give the long name. ### Screenshots Before bug fix : ![screen shot 2016-06-07 at 11 16 14 am](https://cloud.githubusercontent.com/assets/12127192/15847327/a521e768-2ca2-11e6-935c-1114dfabfd76.png) After bug fix : ![screen shot 2016-06-07 at 11 17 15 am](https://cloud.githubusercontent.com/assets/12127192/15847326/a51fc7da-2ca2-11e6-867d-9cd421abc8dc.png) ![screen shot 2016-06-07 at 11 17 31 am](https://cloud.githubusercontent.com/assets/12127192/15847328/a621e50a-2ca2-11e6-8b34-2738c998bb4d.png) ### Questions: * Does the licenses files need update? No. * Is there breaking changes for older versions? No. * Does this needs documentation? No. Author: Sagar Kulkarni Closes #971 from sagarkulkarni3592/ZEPPELIN-961 and squashes the following commits: f7202c0 [Sagar Kulkarni] Merge branch 'master' of https://github.com/apache/incubator-zeppelin into ZEPPELIN-961 ff88f41 [Sagar Kulkarni] Made buttons position dynamic to size of name. a42045e [Sagar Kulkarni] Fixed the input box and button overlap. 9b29d8a [Sagar Kulkarni] Merge branch 'master' of https://github.com/apache/incubator-zeppelin into ZEPPELIN-961 37012ad [Sagar Kulkarni] Made adjustment to the name when window size is changed. 29607e5 [Sagar Kulkarni] ZEPPELIN-961 - Fixed the floating problem as well as styling of buttons. --- .../apache/zeppelin/AbstractZeppelinIT.java | 2 +- .../zeppelin/integration/ZeppelinIT.java | 2 +- .../src/app/notebook/notebook-actionBar.html | 32 +++++++++++-------- zeppelin-web/src/app/notebook/notebook.css | 11 +++++-- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java index e7dba46c946..e78c992fb31 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java @@ -145,7 +145,7 @@ protected void createNewNote() { } protected void deleteTestNotebook(final WebDriver driver) { - driver.findElement(By.xpath("//*[@id='main']/div//h3/span/button[@tooltip='Remove the notebook']")) + driver.findElement(By.xpath(".//*[@id='main']//button[@ng-click='removeNote(note.id)']")) .sendKeys(Keys.ENTER); sleep(1000, true); driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'delete this notebook')]" + diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java index c3d3566f77e..50b67b36d4c 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ZeppelinIT.java @@ -176,7 +176,7 @@ public void testAngularDisplay() throws Exception { waitForText("BindingTest_1_", By.xpath(getParagraphXPath(1) + "//div[@id=\"angularTestButton\"]")); - driver.findElement(By.xpath("//*[@id='main']/div//h3/span/button[@tooltip='Remove the notebook']")) + driver.findElement(By.xpath(".//*[@id='main']//button[@ng-click='removeNote(note.id)']")) .sendKeys(Keys.ENTER); sleep(1000, true); driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'delete this notebook')]" + diff --git a/zeppelin-web/src/app/notebook/notebook-actionBar.html b/zeppelin-web/src/app/notebook/notebook-actionBar.html index 16cb71a26a3..4f7d9841c55 100644 --- a/zeppelin-web/src/app/notebook/notebook-actionBar.html +++ b/zeppelin-web/src/app/notebook/notebook-actionBar.html @@ -13,10 +13,13 @@ -->

    - + -

    {{noteName(note)}}

    - +

    {{noteName(note)}}

    +

    +
    + - + + + - +
    +
    -
    +
    Date: Fri, 24 Jun 2016 02:21:28 -0700 Subject: [PATCH 43/88] [ZEPPELIN-1047] Add filter to handle upper case ### What is this PR for? The notebook filter in navbar does not filter the results of note if note name is given in upper case . ### What type of PR is it? [Bug Fix] ### Todos ### What is the Jira issue? [ZEPPELIN-1047](https://issues.apache.org/jira/browse/ZEPPELIN-1047) ### How should this be tested? 1.Open the notebook filter under navbar -> Notebook menu. 2.Search for any note by giving name in upper case. 3.You should be able to search all the note book names irrespective of case. ### Screenshots (if appropriate) **BEFORE:** http://g.recordit.co/jqo7DYXStI.gif **AFTER:** http://g.recordit.co/uuCT79uEti.gif ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: suvam97 Closes #1066 from suvam97/ZEPPELIN-1047 and squashes the following commits: 6bbbdf1 [suvam97] Merge branch 'master' of https://github.com/apache/zeppelin into ZEPPELIN-1047 86477e3 [suvam97] Merge branch 'master' of https://github.com/apache/zeppelin into ZEPPELIN-1047 5352f27 [suvam97] Removed notebookFilter function 5f176cb [suvam97] Add filter to hamdle upper case --- .../components/navbar/navbar.controller.js | 37 ------------------- .../src/components/navbar/navbar.html | 3 +- 2 files changed, 1 insertion(+), 39 deletions(-) diff --git a/zeppelin-web/src/components/navbar/navbar.controller.js b/zeppelin-web/src/components/navbar/navbar.controller.js index 16209344f56..f13681ef874 100644 --- a/zeppelin-web/src/components/navbar/navbar.controller.js +++ b/zeppelin-web/src/components/navbar/navbar.controller.js @@ -15,43 +15,6 @@ 'use strict'; angular.module('zeppelinWebApp') -.filter('notebookFilter', function() { - return function (notebooks, searchText) - { - if (!searchText) { - return notebooks; - } - - var filteringNote = function(notebooks, filteredNotes) { - _.each(notebooks, function(notebook) { - - if (notebook.name.toLowerCase().indexOf(searchText) !== -1) { - filteredNotes.push(notebook); - return notebook; - } - - if (notebook.children) { - filteringNote(notebook.children, filteredNotes); - } - }); - }; - - return _.filter(notebooks, function(notebook) { - if (notebook.children) { - var filteredNotes = []; - filteringNote(notebook.children, filteredNotes); - - if (filteredNotes.length > 0) { - return filteredNotes; - } - } - - if (notebook.name.toLowerCase().indexOf(searchText) !== -1) { - return notebook; - } - }); - }; -}) .controller('NavCtrl', function($scope, $rootScope, $http, $routeParams, $location, notebookListDataFactory, baseUrlSrv, websocketMsgSrv, arrayOrderingSrv, searchService) { diff --git a/zeppelin-web/src/components/navbar/navbar.html b/zeppelin-web/src/components/navbar/navbar.html index 9ba2cb14d95..cfe1559c6a3 100644 --- a/zeppelin-web/src/components/navbar/navbar.html +++ b/zeppelin-web/src/components/navbar/navbar.html @@ -44,8 +44,7 @@
  • -
  • +
  • From e1b38472235dcb9ab82006757e8d9bb102b6609f Mon Sep 17 00:00:00 2001 From: Hao Xia Date: Thu, 23 Jun 2016 11:19:38 -0700 Subject: [PATCH 44/88] [ZEPPELIN-954] Fix table cell selection problem on second run by properly destroying hot. ### What is this PR for? * Fix table cell selection problem on second run by properly destroying hot. * Also make cells readonly. Previously one were able to paste into them. ### What type of PR is it? [Bug Fix] ### Todos ### What is the Jira issue? * [ZEPPELIN-954] ### How should this be tested? Execute the following paragraph multiple times, and verify the table cells are still selectable. ``` %sh echo %table echo -e "col1\tcol2\tcol3" echo -e "1\t2.1\tabcdefg" ``` Also try to paste anything into a cell to no avail. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Hao Xia Closes #1059 from jasonxh/hao/hot-fix and squashes the following commits: 38d3ef4 [Hao Xia] Use the data argument consistently 1eb7fe4 [Hao Xia] Reuse the table when possible 5bd9502 [Hao Xia] Fix selection problem on second run by properly destroying hot. Also make cells readonly. Previously one were able to paste into them. --- .../paragraph/paragraph.controller.js | 79 +++++++++++-------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 6ab63cda1f9..398191c8e42 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -1221,43 +1221,56 @@ angular.module('zeppelinWebApp') websocketMsgSrv.commitParagraph($scope.paragraph.id, title, text, config, params); }; - var setTable = function(type, data, refresh) { + var setTable = function(data, refresh) { var renderTable = function() { var height = $scope.paragraph.config.graph.height; - angular.element('#p' + $scope.paragraph.id + '_table').css('height', height); - var resultRows = $scope.paragraph.result.rows; - var columnNames = _.pluck($scope.paragraph.result.columnNames, 'name'); - var container = document.getElementById('p' + $scope.paragraph.id + '_table'); + var container = angular.element('#p' + $scope.paragraph.id + '_table').css('height', height).get(0); + var resultRows = data.rows; + var columnNames = _.pluck(data.columnNames, 'name'); + + // on chart type change, destroy table to force reinitialization. + if ($scope.hot && !refresh) { + $scope.hot.destroy(); + $scope.hot = null; + } + + // create table if not exists. + if (!$scope.hot) { + $scope.hot = new Handsontable(container, { + rowHeaders: false, + stretchH: 'all', + sortIndicator: true, + columnSorting: true, + contextMenu: false, + manualColumnResize: true, + manualRowResize: true, + readOnly: true, + readOnlyCellClassName: '', // don't apply any special class so we can retain current styling + fillHandle: false, + fragmentSelection: true, + disableVisualSelection: true, + cells: function (row, col, prop) { + var cellProperties = {}; + cellProperties.renderer = function(instance, td, row, col, prop, value, cellProperties) { + if (!isNaN(value)) { + cellProperties.format = '0,0.[00000]'; + td.style.textAlign = 'left'; + Handsontable.renderers.NumericRenderer.apply(this, arguments); + } else if (value.length > '%html'.length && '%html ' === value.substring(0, '%html '.length)) { + td.innerHTML = value.substring('%html'.length); + } else { + Handsontable.renderers.TextRenderer.apply(this, arguments); + } + }; + return cellProperties; + } + }); + } - var handsontable = new Handsontable(container, { - data: resultRows, + // load data into table. + $scope.hot.updateSettings({ colHeaders: columnNames, - rowHeaders: false, - stretchH: 'all', - sortIndicator: true, - columnSorting: true, - contextMenu: false, - manualColumnResize: true, - manualRowResize: true, - editor: false, - fillHandle: false, - fragmentSelection: true, - disableVisualSelection: true, - cells: function (row, col, prop) { - var cellProperties = {}; - cellProperties.renderer = function(instance, td, row, col, prop, value, cellProperties) { - if (!isNaN(value)) { - cellProperties.format = '0,0.[00000]'; - td.style.textAlign = 'left'; - Handsontable.renderers.NumericRenderer.apply(this, arguments); - } else if (value.length > '%html'.length && '%html ' === value.substring(0, '%html '.length)) { - td.innerHTML = value.substring('%html'.length); - } else { - Handsontable.renderers.TextRenderer.apply(this, arguments); - } - }; - return cellProperties; - } + data: resultRows }); }; From 5975125f18856c11d129621de1914260158ce0e7 Mon Sep 17 00:00:00 2001 From: AhyoungRyu Date: Sat, 25 Jun 2016 12:44:53 -0700 Subject: [PATCH 45/88] [ZEPPELIN-1018] Apply auto "Table of Contents" generator to Zeppelin docs website ### What is this PR for? I added auto TOC(Table of Contents) generator for Zeppelin documentation website. TOC can help people looking through whole contents at a glance and finding what they want quickly. I just added `
    ` to the each documentation header. [`toc`](https://github.com/apache/zeppelin/compare/master...AhyoungRyu:ZEPPELIN-1018?expand=1#diff-85af09fb498a5667ea455391533f945dR3) recognize `

    ` & `

    ` as a title in the docs and it automatically generate TOC. So I set a rule for this work. (I'll write this rule on `docs/CONTRIBUTING.md` or [docs/howtocontributewebsite](https://zeppelin.apache.org/docs/0.6.0-SNAPSHOT/development/howtocontributewebsite.html)). ``` # Level-1 Heading <- Use only for the main title of the page ## Level-2 Heading <- Start with this one ### Level-3 heading <- Only use this one for child of Level-2 toc only recognize Level-2 & Level-3 ``` Please see the below attached screenshot image. ### What type of PR is it? Improvement & Documentation ### Todos * [x] - Add TOC generator * [x] - Apply TOC(`
    `) to every documentation and reorganize each headers(apply the above rule) * [x] - Fix some broken code block in several docs * [x] - Apply TOC to `r.md` (Currently R docs has some duplicated info since [this one](https://github.com/apache/zeppelin/commit/d5e87fb8ba98f08db5b0a4995104ce19f182c678) and [this one](https://github.com/apache/zeppelin/commit/7d6cc7e99154e2d337c11fdf8be1a874ed3e9ada) ) * [x] - Apply TOC to `install.md` after #1010 merged * [x] - Apply TOC to `interpreterinstallation.md` after #1042 merged ### What is the Jira issue? [ZEPPELIN-1018](https://issues.apache.org/jira/browse/ZEPPELIN-1018) ### How should this be tested? 1. Apply this patch and build `docs/` with [this guide](https://github.com/apache/zeppelin/tree/master/docs#build-documentation) 2. Visit some docs page. Then you can see TOC in the header of page. ### Screenshots (if appropriate) - Automatically generated TOC in Spark interpreter docs page screen shot 2016-06-16 at 9 37 18 pm ### Questions: * Does the licenses files need update? No. Actually I used [jekyll-table-of-contents#copyright](https://github.com/ghiculescu/jekyll-table-of-contents#copyright). But I don't need to add a license for this :) * Is there breaking changes for older versions? No * Does this needs documentation? Maybe Author: AhyoungRyu Closes #1031 from AhyoungRyu/ZEPPELIN-1018 and squashes the following commits: e66397b [AhyoungRyu] Apply TOC to interpreterinstallation.md 009579b [AhyoungRyu] Add more info to 'What is the next?' in install.md 04cf501 [AhyoungRyu] Revert 'where to start' section b7cbe5f [AhyoungRyu] Fix typo cf0911c [AhyoungRyu] Rename license file 388f35a [AhyoungRyu] Add jekyll-table-of-contents license info 6394c70 [AhyoungRyu] Fix image path in python.md d00e4b1 [AhyoungRyu] Move interpreter/screenshot/ -> asset/../img/docs-img/ 3ffb383 [AhyoungRyu] Remove duplicated info in r.md & apply toc a03ca99 [AhyoungRyu] Exclude toc.js from pom.xml 3fae7df [AhyoungRyu] Apply auto generated toc to install.md d114a9d [AhyoungRyu] Address @felixcheung feedback 6a788fe [AhyoungRyu] Resize TOC tab indent 6760c00 [AhyoungRyu] Apply auto TOC to all of docs under docs/storage/ fbde57f [AhyoungRyu] Apply auto TOC to all of docs under docs/quickstart/ db76eb6 [AhyoungRyu] Apply auto TOC to all of docs under docs/install/ f35db47 [AhyoungRyu] Apply auto TOC to all of docs under docs/displaysystem/ b05365f [AhyoungRyu] Apply auto TOC to all of docs under docs/rest-api/ 163691c [AhyoungRyu] Apply auto TOC to all of docs under docs/manual/ bef398e [AhyoungRyu] Apply auto TOC to all of docs under docs/development/ 9c5f76b [AhyoungRyu] Apply auto TOC to all of docs under docs/interpreter/ 587d4ba [AhyoungRyu] Apply auto TOC to all of docs under docs/security/ 1f10b97 [AhyoungRyu] Change toc configuration 78dca9e [AhyoungRyu] Add toc.js for auto generating TOC --- LICENSE | 8 +- docs/_includes/themes/zeppelin/default.html | 1 + docs/assets/themes/zeppelin/css/style.css | 14 ++ .../zeppelin/img/docs-img}/backtoscala.png | Bin .../themes/zeppelin/img/docs-img}/knitgeo.png | Bin .../zeppelin/img/docs-img}/knitmotion.png | Bin .../zeppelin/img/docs-img}/knitstock.png | Bin .../img/docs-img}/pythonMatplotlib.png | Bin .../zeppelin/img/docs-img}/repl2plus2.png | Bin .../zeppelin/img/docs-img}/replhead.png | Bin .../zeppelin/img/docs-img}/replhist.png | Bin .../zeppelin/img/docs-img}/sparkrfaithful.png | Bin .../themes/zeppelin/img/docs-img}/varr1.png | Bin .../themes/zeppelin/img/docs-img}/varr2.png | Bin .../zeppelin/img/docs-img}/varscala.png | Bin docs/assets/themes/zeppelin/js/docs.js | 4 + docs/assets/themes/zeppelin/js/toc.js | 98 ++++++++++++++ docs/development/howtocontribute.md | 25 ++-- docs/development/howtocontributewebsite.md | 41 +++--- .../development/writingzeppelininterpreter.md | 66 ++++----- docs/displaysystem/back-end-angular.md | 29 ++-- docs/displaysystem/basicdisplaysystem.md | 4 + docs/displaysystem/front-end-angular.md | 25 ++-- docs/install/install.md | 28 ++-- docs/install/upgrade.md | 10 +- docs/install/virtual_machine.md | 28 ++-- docs/interpreter/alluxio.md | 6 +- docs/interpreter/cassandra.md | 4 +- docs/interpreter/elasticsearch.md | 6 +- docs/interpreter/flink.md | 6 +- docs/interpreter/geode.md | 30 +++-- docs/interpreter/hbase.md | 12 +- docs/interpreter/hdfs.md | 13 +- docs/interpreter/hive.md | 10 +- docs/interpreter/ignite.md | 14 +- docs/interpreter/jdbc.md | 57 ++++---- docs/interpreter/lens.md | 12 +- docs/interpreter/livy.md | 21 +-- docs/interpreter/markdown.md | 8 +- docs/interpreter/postgresql.md | 46 ++++--- docs/interpreter/python.md | 12 +- docs/interpreter/r.md | 114 +++++++--------- docs/interpreter/scalding.md | 19 +-- docs/interpreter/spark.md | 22 +-- docs/manual/dynamicform.md | 22 +-- docs/manual/dynamicinterpreterload.md | 25 ++-- docs/manual/interpreterinstallation.md | 2 + docs/manual/interpreters.md | 22 +-- docs/manual/notebookashomepage.md | 125 +++++++++--------- docs/manual/publish.md | 15 ++- docs/quickstart/explorezeppelinui.md | 26 ++-- docs/quickstart/tutorial.md | 15 ++- docs/rest-api/rest-configuration.md | 19 ++- docs/rest-api/rest-interpreter.md | 35 ++--- docs/rest-api/rest-notebook.md | 71 ++++------ docs/security/authentication.md | 15 ++- docs/security/interpreter_authorization.md | 2 + docs/security/notebook_authorization.md | 7 +- docs/security/shiroauthentication.md | 26 ++-- docs/storage/storage.md | 23 ++-- licenses/LICENSE-jekyll-table-of-contents | 19 +++ pom.xml | 3 + 62 files changed, 745 insertions(+), 520 deletions(-) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/backtoscala.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/knitgeo.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/knitmotion.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/knitstock.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/pythonMatplotlib.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/repl2plus2.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/replhead.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/replhist.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/sparkrfaithful.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/varr1.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/varr2.png (100%) rename docs/{interpreter/screenshots => assets/themes/zeppelin/img/docs-img}/varscala.png (100%) create mode 100755 docs/assets/themes/zeppelin/js/toc.js create mode 100644 licenses/LICENSE-jekyll-table-of-contents diff --git a/LICENSE b/LICENSE index 67a5b7934ae..71d3c525f1b 100644 --- a/LICENSE +++ b/LICENSE @@ -235,7 +235,13 @@ The text of each license is also included at licenses/LICENSE-[project]-[version (The MIT License) jekyll-bootstrap 0.3.0 (https://github.com/plusjade/jekyll-bootstrap) - https://github.com/plusjade/jekyll-bootstrap (The MIT License) jekyll 1.3.0 (http://jekyllrb.com/) - https://github.com/jekyll/jekyll/blob/v1.3.0/LICENSE - +======================================================================== +MIT-style licenses +======================================================================== +The following components are provided under the MIT-style license. See project link for details. +The text of each license is also included at licenses/LICENSE-[project]-[version].txt. + + (MIT Style) jekyll-table-of-contents (https://github.com/ghiculescu/jekyll-table-of-contents) - https://github.com/ghiculescu/jekyll-table-of-contents/blob/master/LICENSE.txt ======================================================================== Apache licenses diff --git a/docs/_includes/themes/zeppelin/default.html b/docs/_includes/themes/zeppelin/default.html index eb99b9bd0fd..cd07602ec90 100644 --- a/docs/_includes/themes/zeppelin/default.html +++ b/docs/_includes/themes/zeppelin/default.html @@ -33,6 +33,7 @@ + diff --git a/docs/assets/themes/zeppelin/css/style.css b/docs/assets/themes/zeppelin/css/style.css index a5507d31311..8ab197c412e 100644 --- a/docs/assets/themes/zeppelin/css/style.css +++ b/docs/assets/themes/zeppelin/css/style.css @@ -545,6 +545,20 @@ a.anchor { a.anchorjs-link:hover { text-decoration: none; } +/* Table of Contents(TOC) */ +#toc { + padding-top: 12px; + padding-bottom: 12px; +} + +#toc ul { + margin-left: -14px; +} + +#toc ul ul { + margin-left: -18px; +} + /* Custom, iPhone Retina */ @media only screen and (max-width: 480px) { .jumbotron h1 { diff --git a/docs/interpreter/screenshots/backtoscala.png b/docs/assets/themes/zeppelin/img/docs-img/backtoscala.png similarity index 100% rename from docs/interpreter/screenshots/backtoscala.png rename to docs/assets/themes/zeppelin/img/docs-img/backtoscala.png diff --git a/docs/interpreter/screenshots/knitgeo.png b/docs/assets/themes/zeppelin/img/docs-img/knitgeo.png similarity index 100% rename from docs/interpreter/screenshots/knitgeo.png rename to docs/assets/themes/zeppelin/img/docs-img/knitgeo.png diff --git a/docs/interpreter/screenshots/knitmotion.png b/docs/assets/themes/zeppelin/img/docs-img/knitmotion.png similarity index 100% rename from docs/interpreter/screenshots/knitmotion.png rename to docs/assets/themes/zeppelin/img/docs-img/knitmotion.png diff --git a/docs/interpreter/screenshots/knitstock.png b/docs/assets/themes/zeppelin/img/docs-img/knitstock.png similarity index 100% rename from docs/interpreter/screenshots/knitstock.png rename to docs/assets/themes/zeppelin/img/docs-img/knitstock.png diff --git a/docs/interpreter/screenshots/pythonMatplotlib.png b/docs/assets/themes/zeppelin/img/docs-img/pythonMatplotlib.png similarity index 100% rename from docs/interpreter/screenshots/pythonMatplotlib.png rename to docs/assets/themes/zeppelin/img/docs-img/pythonMatplotlib.png diff --git a/docs/interpreter/screenshots/repl2plus2.png b/docs/assets/themes/zeppelin/img/docs-img/repl2plus2.png similarity index 100% rename from docs/interpreter/screenshots/repl2plus2.png rename to docs/assets/themes/zeppelin/img/docs-img/repl2plus2.png diff --git a/docs/interpreter/screenshots/replhead.png b/docs/assets/themes/zeppelin/img/docs-img/replhead.png similarity index 100% rename from docs/interpreter/screenshots/replhead.png rename to docs/assets/themes/zeppelin/img/docs-img/replhead.png diff --git a/docs/interpreter/screenshots/replhist.png b/docs/assets/themes/zeppelin/img/docs-img/replhist.png similarity index 100% rename from docs/interpreter/screenshots/replhist.png rename to docs/assets/themes/zeppelin/img/docs-img/replhist.png diff --git a/docs/interpreter/screenshots/sparkrfaithful.png b/docs/assets/themes/zeppelin/img/docs-img/sparkrfaithful.png similarity index 100% rename from docs/interpreter/screenshots/sparkrfaithful.png rename to docs/assets/themes/zeppelin/img/docs-img/sparkrfaithful.png diff --git a/docs/interpreter/screenshots/varr1.png b/docs/assets/themes/zeppelin/img/docs-img/varr1.png similarity index 100% rename from docs/interpreter/screenshots/varr1.png rename to docs/assets/themes/zeppelin/img/docs-img/varr1.png diff --git a/docs/interpreter/screenshots/varr2.png b/docs/assets/themes/zeppelin/img/docs-img/varr2.png similarity index 100% rename from docs/interpreter/screenshots/varr2.png rename to docs/assets/themes/zeppelin/img/docs-img/varr2.png diff --git a/docs/interpreter/screenshots/varscala.png b/docs/assets/themes/zeppelin/img/docs-img/varscala.png similarity index 100% rename from docs/interpreter/screenshots/varscala.png rename to docs/assets/themes/zeppelin/img/docs-img/varscala.png diff --git a/docs/assets/themes/zeppelin/js/docs.js b/docs/assets/themes/zeppelin/js/docs.js index 1d2d002344c..343f6e85d57 100644 --- a/docs/assets/themes/zeppelin/js/docs.js +++ b/docs/assets/themes/zeppelin/js/docs.js @@ -118,6 +118,10 @@ $(function() { maybeScrollToHash(); }); + $(document).ready(function() { + $('#toc').toc(); + }); + // Scroll now too in case we had opened the page on a hash, but wait a bit because some browsers // will try to do *their* initial scroll after running the onReady handler. $(window).load(function() { setTimeout(function() { maybeScrollToHash(); }, 25); }); diff --git a/docs/assets/themes/zeppelin/js/toc.js b/docs/assets/themes/zeppelin/js/toc.js new file mode 100755 index 00000000000..8977fff2137 --- /dev/null +++ b/docs/assets/themes/zeppelin/js/toc.js @@ -0,0 +1,98 @@ +// https://github.com/ghiculescu/jekyll-table-of-contents +(function($){ + $.fn.toc = function(options) { + var defaults = { + noBackToTopLinks: false, + title: '', + minimumHeaders: 2, + headers: 'h2, h3', + listType: 'ul', // values: [ol|ul] + showEffect: 'none', // values: [show|slideDown|fadeIn|none] + showSpeed: '0', // set to 0 to deactivate effect + classes: { list: '', + item: '' + } + }, + settings = $.extend(defaults, options); + + function fixedEncodeURIComponent (str) { + return encodeURIComponent(str).replace(/[!'()*]/g, function(c) { + return '%' + c.charCodeAt(0).toString(16); + }); + } + + function createLink (header) { + var innerText = (header.textContent === undefined) ? header.innerText : header.textContent; + return "" + innerText + ""; + } + + var headers = $(settings.headers).filter(function() { + // get all headers with an ID + var previousSiblingName = $(this).prev().attr( "name" ); + if (!this.id && previousSiblingName) { + this.id = $(this).attr( "id", previousSiblingName.replace(/\./g, "-") ); + } + return this.id; + }), output = $(this); + if (!headers.length || headers.length < settings.minimumHeaders || !output.length) { + $(this).hide(); + return; + } + + if (0 === settings.showSpeed) { + settings.showEffect = 'none'; + } + + var render = { + show: function() { output.hide().html(html).show(settings.showSpeed); }, + slideDown: function() { output.hide().html(html).slideDown(settings.showSpeed); }, + fadeIn: function() { output.hide().html(html).fadeIn(settings.showSpeed); }, + none: function() { output.html(html); } + }; + + var get_level = function(ele) { return parseInt(ele.nodeName.replace("H", ""), 10); }; + var highest_level = headers.map(function(_, ele) { return get_level(ele); }).get().sort()[0]; + var return_to_top = ' '; + + var level = get_level(headers[0]), + this_level, + html = settings.title + " <" +settings.listType + " class=\"" + settings.classes.list +"\">"; + headers.on('click', function() { + if (!settings.noBackToTopLinks) { + window.location.hash = this.id; + } + }) + .addClass('clickable-header') + .each(function(_, header) { + this_level = get_level(header); + if (!settings.noBackToTopLinks && this_level === highest_level) { + $(header).addClass('top-level-header').after(return_to_top); + } + if (this_level === level) // same level as before; same indenting + html += "
  • " + createLink(header); + else if (this_level <= level){ // higher level than before; end parent ol + for(i = this_level; i < level; i++) { + html += "
    • " + } + html += "
  • " + createLink(header); + } + else if (this_level > level) { // lower level than before; expand the previous to contain a ol + for(i = this_level; i > level; i--) { + html += "<" + settings.listType + " class=\"" + settings.classes.list +"\">" + + "
  • " + } + html += createLink(header); + } + level = this_level; // update for the next one + }); + html += ""; + if (!settings.noBackToTopLinks) { + $(document).on('click', '.back-to-top', function() { + $(window).scrollTop(0); + window.location.hash = ''; + }); + } + + render[settings.showEffect](); + }; +})(jQuery); diff --git a/docs/development/howtocontribute.md b/docs/development/howtocontribute.md index 7b3ee0cfe6b..2d3842a2257 100644 --- a/docs/development/howtocontribute.md +++ b/docs/development/howtocontribute.md @@ -7,8 +7,9 @@ group: development # Contributing to Apache Zeppelin ( Code ) -## IMPORTANT -Apache Zeppelin is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software. +
    + +> **NOTE :** Apache Zeppelin is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software. Any contributions to Zeppelin (Source code, Documents, Image, Website) means you agree with license all your contributions as Apache2 License. ## Setting up @@ -22,7 +23,7 @@ Since Zeppelin uses Git for it's SCM system, you need git client installed in yo You are free to use whatever IDE you prefer, or your favorite command line editor. -### Build Tools +#### Build Tools To build the code, install @@ -46,10 +47,10 @@ You may also want to develop against a specific branch. For example, for branch- git clone -b branch-0.5.6 git://git.apache.org/zeppelin.git zeppelin ``` -#### Fork repository -If you want not only build Zeppelin but also make any changes, then you need fork [Zeppelin github mirror repository](https://github.com/apache/zeppelin) and make a pull request. +Apache Zeppelin follows [Fork & Pull](https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) as a source control workflow. +If you want to not only build Zeppelin but also make any changes, then you need to fork [Zeppelin github mirror repository](https://github.com/apache/zeppelin) and make a pull request. -###Build +### Build ``` mvn install @@ -67,6 +68,8 @@ To build with specific spark / hadoop version mvn install -Dspark.version=x.x.x -Dhadoop.version=x.x.x ``` +For the further + ### Run Zeppelin server in development mode ``` @@ -88,21 +91,19 @@ Server will be run on [http://localhost:8080](http://localhost:8080). Some portions of the Zeppelin code are generated by [Thrift](http://thrift.apache.org). For most Zeppelin changes, you don't need to worry about this. But if you modify any of the Thrift IDL files (e.g. zeppelin-interpreter/src/main/thrift/*.thrift), then you also need to regenerate these files and submit their updated version as part of your patch. -To regenerate the code, install **thrift-0.9.0** and change directory into Zeppelin source directory. and then run following command +To regenerate the code, install **thrift-0.9.2** and change directory into Zeppelin source directory. and then run following command ``` thrift -out zeppelin-interpreter/src/main/java/ --gen java zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift ``` - -## JIRA -Zeppelin manages its issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN) - ## Where to Start -You can find issues for [beginner](https://issues.apache.org/jira/browse/ZEPPELIN-924?jql=project%20%3D%20ZEPPELIN%20and%20status%20%3D%20Open%20and%20labels%20in%20\(beginner%2C%20newbie\)). +You can find issues for beginner & newbie ## Stay involved Contributors should join the Zeppelin mailing lists. * [dev@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/) is for people who want to contribute code to Zeppelin. [subscribe](mailto:dev-subscribe@zeppelin.apache.org?subject=send this email to subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.apache.org?subject=send this email to unsubscribe), [archives](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/) + +If you have any issues, create a ticket in [JIRA](https://issues.apache.org/jira/browse/ZEPPELIN). diff --git a/docs/development/howtocontributewebsite.md b/docs/development/howtocontributewebsite.md index f56b8e3ee1f..0db15551585 100644 --- a/docs/development/howtocontributewebsite.md +++ b/docs/development/howtocontributewebsite.md @@ -7,49 +7,52 @@ group: development # Contributing to Apache Zeppelin ( Website ) -## IMPORTANT -Apache Zeppelin is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software. -Any contribution to Zeppelin (Source code, Documents, Image, Website) means you agree license all your contributions as Apache2 License. +
    -## Modifying the website +This page will give you an overview of how to build and contribute to the documentation of Apache Zeppelin. +The online documentation at [zeppelin.apache.org](https://zeppelin.apache.org/docs/latest/) is also generated from the files found here. -#### Getting the source code -Website is hosted in 'master' branch under `/docs/` dir. +> **NOTE :** Apache Zeppelin is an [Apache2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Software. +Any contributions to Zeppelin (Source code, Documents, Image, Website) means you agree with license all your contributions as Apache2 License. -First of all, you need the website source code. The official location of mirror for Zeppelin is [http://git.apache.org/zeppelin.git](http://git.apache.org/zeppelin.git). +## Getting the source code +First of all, you need Zeppelin source code. The official location of Zeppelin is [http://git.apache.org/zeppelin.git](http://git.apache.org/zeppelin.git). +Documentation website is hosted in 'master' branch under `/docs/` dir. + +### git access +First of all, you need the website source code. The official location of mirror for Zeppelin is [http://git.apache.org/zeppelin.git](http://git.apache.org/zeppelin.git). Get the source code on your development machine using git. ``` git clone git://git.apache.org/zeppelin.git cd docs ``` +Apache Zeppelin follows [Fork & Pull](https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) as a source control workflow. +If you want to not only build Zeppelin but also make any changes, then you need to fork [Zeppelin github mirror repository](https://github.com/apache/zeppelin) and make a pull request. -#### Build - -To build, you'll need to install some prerequisites. Please check 'Build documentation' section in [docs/README.md](https://github.com/apache/zeppelin/blob/master/docs/README.md#build-documentation). +### Build -#### Run website in development mode +You'll need to install some prerequisites to build the code. Please check [Build documentation](https://github.com/apache/zeppelin/blob/master/docs/README.md#build-documentation) section in [docs/README.md](https://github.com/apache/zeppelin/blob/master/docs/README.md). -While you're modifying website, you'll want to see preview of it. Please check 'Run website' section in [docs/README.md](https://github.com/apache/zeppelin/blob/master/docs/README.md#run-website). +### Run website in development mode -You'll be able to access it on [http://localhost:4000](http://localhost:4000) with your web browser. +While you're modifying website, you might want to see preview of it. Please check [Run website](https://github.com/apache/zeppelin/blob/master/docs/README.md#run-website) section in [docs/README.md](https://github.com/apache/zeppelin/blob/master/docs/README.md). +Then you'll be able to access it on [http://localhost:4000](http://localhost:4000) with your web browser. -#### Making a Pull Request +### Making a Pull Request When you are ready, just make a pull-request. ## Alternative way -You can directly edit .md files in `/docs/` dir at github's web interface and make pull-request immediatly. - - -## JIRA -Zeppelin manages its issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN) +You can directly edit `.md` files in `/docs/` directory at the web interface of github and make pull-request immediatly. ## Stay involved Contributors should join the Zeppelin mailing lists. * [dev@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/) is for people who want to contribute code to Zeppelin. [subscribe](mailto:dev-subscribe@zeppelin.apache.org?subject=send this email to subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.apache.org?subject=send this email to unsubscribe), [archives](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/) + +If you have any issues, create a ticket in [JIRA](https://issues.apache.org/jira/browse/ZEPPELIN). diff --git a/docs/development/writingzeppelininterpreter.md b/docs/development/writingzeppelininterpreter.md index f3412116527..7e7f4ef2653 100644 --- a/docs/development/writingzeppelininterpreter.md +++ b/docs/development/writingzeppelininterpreter.md @@ -19,21 +19,25 @@ limitations under the License. --> {% include JB/setup %} -### What is Zeppelin Interpreter +# Writing a New Interpreter -Zeppelin Interpreter is a language backend. For example to use scala code in Zeppelin, you need scala interpreter. -Every Interpreter belongs to an InterpreterGroup. +
    + +## What is Apache Zeppelin Interpreter + +Apache Zeppelin Interpreter is a language backend. For example to use scala code in Zeppelin, you need a scala interpreter. +Every Interpreters belongs to an **InterpreterGroup**. Interpreters in the same InterpreterGroup can reference each other. For example, SparkSqlInterpreter can reference SparkInterpreter to get SparkContext from it while they're in the same group. -InterpreterSetting is configuration of a given InterpreterGroup and a unit of start/stop interpreter. -All Interpreters in the same InterpreterSetting are launched in a single, separate JVM process. The Interpreter communicates with Zeppelin engine via thrift. +[InterpreterSetting](https://github.com/apache/zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java) is configuration of a given [InterpreterGroup](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java) and a unit of start/stop interpreter. +All Interpreters in the same InterpreterSetting are launched in a single, separate JVM process. The Interpreter communicates with Zeppelin engine via **[Thrift](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift)**. -In 'Separate Interpreter for each note' mode, new Interpreter instance will be created per notebook. But it still runs on the same JVM while they're in the same InterpreterSettings. +In 'Separate Interpreter(scoped / isolated) for each note' mode which you can see at the **Interpreter Setting** menu when you create a new interpreter, new interpreter instance will be created per notebook. But it still runs on the same JVM while they're in the same InterpreterSettings. -### Make your own Interpreter +## Make your own Interpreter Creating a new interpreter is quite simple. Just extend [org.apache.zeppelin.interpreter](https://github.com/apache/zeppelin/blob/master/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java) abstract class and implement some methods. You can include `org.apache.zeppelin:zeppelin-interpreter:[VERSION]` artifact in your build system. And you should your jars under your interpreter directory with specific directory name. Zeppelin server reads interpreter directories recursively and initializes interpreters including your own interpreter. @@ -91,18 +95,18 @@ The name of the interpreter is what you later write to identify a paragraph whic some interpreter specific code... ``` -### Programming Languages for Interpreter +## Programming Languages for Interpreter If the interpreter uses a specific programming language ( like Scala, Python, SQL ), it is generally recommended to add a syntax highlighting supported for that to the notebook paragraph editor. To check out the list of languages supported, see the `mode-*.js` files under `zeppelin-web/bower_components/ace-builds/src-noconflict` or from [github.com/ajaxorg/ace-builds](https://github.com/ajaxorg/ace-builds/tree/master/src-noconflict). If you want to add a new set of syntax highlighting, -1. Add the `mode-*.js` file to `zeppelin-web/bower.json` ( when built, `zeppelin-web/src/index.html` will be changed automatically. ). -2. Add to the list of `editorMode` in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js` - it follows the pattern 'ace/mode/x' where x is the name. -3. Add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` located in `zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js`. +1. Add the `mode-*.js` file to [zeppelin-web/bower.json](https://github.com/apache/zeppelin/blob/master/zeppelin-web/bower.json) ( when built, [zeppelin-web/src/index.html](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/index.html) will be changed automatically. ). +2. Add to the list of `editorMode` in [zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js) - it follows the pattern 'ace/mode/x' where x is the name. +3. Add to the code that checks for `%` prefix and calls `session.setMode(editorMode.x)` in `setParagraphMode` located in [zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js). -### Install your interpreter binary +## Install your interpreter binary Once you have built your interpreter, you can place it under the interpreter directory with all its dependencies. @@ -110,7 +114,7 @@ Once you have built your interpreter, you can place it under the interpreter dir [ZEPPELIN_HOME]/interpreter/[INTERPRETER_NAME]/ ``` -### Configure your interpreter +## Configure your interpreter To configure your interpreter you need to follow these steps: @@ -119,12 +123,12 @@ To configure your interpreter you need to follow these steps: Property value is comma separated [INTERPRETER\_CLASS\_NAME]. For example, -``` - - zeppelin.interpreters - org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,com.me.MyNewInterpreter - -``` + ``` + + zeppelin.interpreters + org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,com.me.MyNewInterpreter + + ``` 2. Add your interpreter to the [default configuration](https://github.com/apache/zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `zeppelin-site.xml`. @@ -133,11 +137,11 @@ To configure your interpreter you need to follow these steps: 4. In the interpreter page, click the `+Create` button and configure your interpreter properties. Now you are done and ready to use your interpreter. -Note that the interpreters released with zeppelin have a [default configuration](https://github.com/apache/zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `conf/zeppelin-site.xml`. +> **Note :** Interpreters released with zeppelin have a [default configuration](https://github.com/apache/zeppelin/blob/master/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java#L397) which is used when there is no `conf/zeppelin-site.xml`. -### Use your interpreter +## Use your interpreter -#### 0.5.0 +### 0.5.0 Inside of a notebook, `%[INTERPRETER_NAME]` directive will call your interpreter. Note that the first interpreter configuration in zeppelin.interpreters will be the default one. @@ -150,8 +154,7 @@ val a = "My interpreter" println(a) ``` -
    -#### 0.6.0 and later +### 0.6.0 and later Inside of a notebook, `%[INTERPRETER_GROUP].[INTERPRETER_NAME]` directive will call your interpreter. Note that the first interpreter configuration in zeppelin.interpreters will be the default one. @@ -192,7 +195,7 @@ You can only omit your interpreter group when your interpreter group is selected codes for myintp2 ``` -### Examples +## Examples Checkout some interpreters released with Zeppelin by default. @@ -201,15 +204,16 @@ Checkout some interpreters released with Zeppelin by default. - [shell](https://github.com/apache/zeppelin/tree/master/shell) - [jdbc](https://github.com/apache/zeppelin/tree/master/jdbc) -### Contributing a new Interpreter to Zeppelin releases +## Contributing a new Interpreter to Zeppelin releases We welcome contribution to a new interpreter. Please follow these few steps: - - First, check out the general contribution guide [here](./howtocontributewebsite.html). - - Follow the steps in "Make your own Interpreter" section above. - - Add your interpreter as in the "Configure your interpreter" section above; also add it to the example template [zeppelin-site.xml.template](https://github.com/apache/zeppelin/blob/master/conf/zeppelin-site.xml.template). - - Add tests! They are run by Travis for all changes and it is important that they are self-contained. + - First, check out the general contribution guide [here](https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md). + - Follow the steps in [Make your own Interpreter](#make-your-own-interpreter) section above. + - Add your interpreter as in the [Configure your interpreter](#configure-your-interpreter) section above; also add it to the example template [zeppelin-site.xml.template](https://github.com/apache/zeppelin/blob/master/conf/zeppelin-site.xml.template). + - Add tests! They are run by [Travis](https://travis-ci.org/apache/zeppelin) for all changes and it is important that they are self-contained. - Include your interpreter as a module in [`pom.xml`](https://github.com/apache/zeppelin/blob/master/pom.xml). - Add documentation on how to use your interpreter under `docs/interpreter/`. Follow the Markdown style as this [example](https://github.com/apache/zeppelin/blob/master/docs/interpreter/elasticsearch.md). Make sure you list config settings and provide working examples on using your interpreter in code boxes in Markdown. Link to images as appropriate (images should go to `docs/assets/themes/zeppelin/img/docs-img/`). And add a link to your documentation in the navigation menu (`docs/_includes/themes/zeppelin/_navigation.html`). - Most importantly, ensure licenses of the transitive closure of all dependencies are list in [license file](https://github.com/apache/zeppelin/blob/master/zeppelin-distribution/src/bin_license/LICENSE). - - Commit your changes and open a Pull Request on the project [Mirror on GitHub](https://github.com/apache/zeppelin); check to make sure Travis CI build is passing. + - Commit your changes and open a [Pull Request](https://github.com/apache/zeppelin/pulls) on the project [Mirror on GitHub](https://github.com/apache/zeppelin); check to make sure Travis CI build is passing. + \ No newline at end of file diff --git a/docs/displaysystem/back-end-angular.md b/docs/displaysystem/back-end-angular.md index 4a06a08ffea..d84a033d6fd 100644 --- a/docs/displaysystem/back-end-angular.md +++ b/docs/displaysystem/back-end-angular.md @@ -19,15 +19,17 @@ limitations under the License. --> {% include JB/setup %} +# Back-end Angular API in Apache Zeppelin -## Back-end Angular API in Zeppelin +
    -Angular display system treats output as a view template for [AngularJS](https://angularjs.org/). -It compiles templates and displays them inside of Zeppelin. +## Overview -Zeppelin provides a gateway between your interpreter and your compiled **AngularJS view** templates. +Angular display system treats output as a view template for [AngularJS](https://angularjs.org/). +It compiles templates and displays them inside of Apache Zeppelin. Zeppelin provides a gateway between your interpreter and your compiled **AngularJS view** templates. Therefore, you can not only update scope variables from your interpreter but also watch them in the interpreter, which is JVM process. +## Basic Usage ### Print AngularJS view To use angular display system, you should start with `%angular`. @@ -94,30 +96,25 @@ When the button is clicked, you'll see both `run` and `numWatched` are increment ## Let's make it Simpler and more Intuitive In this section, we will introduce a simpler and more intuitive way of using **Angular Display System** in Zeppelin. -### How can we use it? Here are some usages. -#### Import - -##### - In notebook scope +### Import ```scala +// In notebook scope import org.apache.zeppelin.display.angular.notebookscope._ import AngularElem._ -``` -##### - In paragraph scope -```scala +// In paragraph scope import org.apache.zeppelin.display.angular.paragraphscope._ import AngularElem._ ``` - -#### Display Element +### Display Element ```scala // automatically convert to string and print with %angular display system directive in front.
    .display ``` -#### Event Handler +### Event Handler ```scala // on click
    .onClick(() => { @@ -135,7 +132,7 @@ import AngularElem._ }).display ``` -#### Bind Model +### Bind Model ```scala // bind model
    .model("myModel").display @@ -144,7 +141,7 @@ import AngularElem._
    .model("myModel", initialValue).display ``` -#### Interact with Model +### Interact with Model ```scala // read model AngularModel("myModel")() diff --git a/docs/displaysystem/basicdisplaysystem.md b/docs/displaysystem/basicdisplaysystem.md index 3821f1d9ccf..732408dfd76 100644 --- a/docs/displaysystem/basicdisplaysystem.md +++ b/docs/displaysystem/basicdisplaysystem.md @@ -19,6 +19,10 @@ limitations under the License. --> {% include JB/setup %} +# Basic Display System in Apache Zeppelin + +
    + ## Text By default, Apache Zeppelin prints interpreter responce as a plain text using `text` display system. diff --git a/docs/displaysystem/front-end-angular.md b/docs/displaysystem/front-end-angular.md index c78b1b4c52d..773b067d255 100644 --- a/docs/displaysystem/front-end-angular.md +++ b/docs/displaysystem/front-end-angular.md @@ -19,18 +19,18 @@ limitations under the License. --> {% include JB/setup %} +# Front-end Angular API in Apache Zeppelin -## Front-end Angular API in Zeppelin +
    -In addition to the back-end API to handle Angular objects binding, Zeppelin also exposes a simple AngularJS **`z`** object on the front-end side to expose the same capabilities. +## Basic Usage +In addition to the [back-end API](./back-end-angular.html) to handle Angular objects binding, Apache Zeppelin also exposes a simple AngularJS **z** object on the front-end side to expose the same capabilities. +This **z** object is accessible in the Angular isolated scope for each paragraph. -This **`z`** object is accessible in the Angular isolated scope for each paragraph. -
    ### Bind / Unbind Variables -Through the **`z`**, you can bind / unbind variables to **AngularJS view** - +Through the **`z`**, you can bind / unbind variables to **AngularJS view**. Bind a value to an angular object and a **mandatory** target paragraph: ```html @@ -68,9 +68,10 @@ Unbind/remove a value from angular object and a **mandatory** target paragraph: The signature for the **`z.angularBind() / z.angularUnbind()`** functions are: ```javascript - +// Bind z.angularBind(angularObjectName, angularObjectValue, paragraphId); +// Unbind z.angularUnbind(angularObjectName, angularObjectValue, paragraphId); ``` @@ -100,24 +101,24 @@ You can also trigger paragraph execution by calling **`z.runParagraph()`** funct
    -### Overriding dynamic form with Angular Object +## Overriding dynamic form with Angular Object The front-end Angular Interaction API has been designed to offer richer form capabilities and variable binding. With the existing **Dynamic Form** system you can already create input text, select and checkbox forms but the choice is rather limited and the look & feel cannot be changed. The idea is to create a custom form using plain HTML/AngularJS code and bind actions on this form to push/remove Angular variables to targeted paragraphs using this new API. -Consequently if you use the **Dynamic Form** syntax in a paragraph and there is a bound Angular object having the same name as the _${formName}_, the Angular object will have higher priority and the **Dynamic Form** will not be displayed. Example: +Consequently if you use the **Dynamic Form** syntax in a paragraph and there is a bound Angular object having the same name as the `${formName}`, the Angular object will have higher priority and the **Dynamic Form** will not be displayed. Example:
    -### Feature matrix comparison +## Feature matrix comparison -How does the front-end AngularJS API compares to the back-end API ? Below is a comparison matrix for both APIs: +How does the front-end AngularJS API compares to the [back-end API](./back-end-angular.html) ? Below is a comparison matrix for both APIs: - +
    diff --git a/docs/install/install.md b/docs/install/install.md index 55741dcf1cb..bcc93da224d 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -23,15 +23,7 @@ limitations under the License. Welcome to your first trial to explore Apache Zeppelin! This page will help you to get started and here is the list of topics covered. -* [Installation](#installation) - * [Downloading Binary Package](#downloading-binary-package) - * [Building from Source](#building-from-source) -* [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) - * [Start Zeppelin](#start-zeppelin) - * [Stop Zeppelin](#stop-zeppelin) - * [(Optional) Start Apache Zeppelin with a service manager](#optional-start-apache-zeppelin-with-a-service-manager) -* [What is the next?](#what-is-the-next) -* [Apache Zeppelin Configuration](#apache-zeppelin-configuration) +
    ## Installation @@ -52,7 +44,7 @@ Apache Zeppelin officially supports and is tested on next environments.
    Actions
    -There are two options to install Apache Zeppelin on your machine. One is [downloading prebuild binary package](#downloading-binary-package) from the archive. +There are two options to install Apache Zeppelin on your machine. One is [downloading pre-built binary package](#downloading-binary-package) from the archive. You can download not only the latest stable version but also the older one if you need. The other option is [building from the source](#building-from-source). Although it can be unstable somehow since it is on development status, you can explore newly added feature and change it as you want. @@ -181,10 +173,18 @@ exec bin/zeppelin-daemon.sh upstart ## What is the next? Congratulation on your successful Apache Zeppelin installation! Here are two next steps you might need. - * For an in-depth overview of Apache Zeppelin UI, head to [Explore Apache Zeppelin UI](../quickstart/explorezeppelinui.html) - * After getting familiar with Apache Zeppelin UI, have fun with a short walk-through [Tutorial](../quickstart/tutorial.html) that uses Apache Spark backend - * If you need more configuration setting for Apache Zeppelin, jump to the next section: [Apache Zeppelin Configuration](#apache-zeppelin-configuration) - +#### If you are new to Apache Zeppelin + * For an in-depth overview of Apache Zeppelin UI, head to [Explore Apache Zeppelin UI](../quickstart/explorezeppelinui.html). + * After getting familiar with Apache Zeppelin UI, have fun with a short walk-through [Tutorial](../quickstart/tutorial.html) that uses Apache Spark backend. + * If you need more configuration setting for Apache Zeppelin, jump to the next section: [Apache Zeppelin Configuration](#apache-zeppelin-configuration). + +#### If you need more information about Spark or JDBC interpreter setting + * Apache Zeppelin provides deep integration with [Apache Spark](http://spark.apache.org/). For the further informtation, see [Spark Interpreter for Apache Zeppelin](../interpreter/spark.html). + * Also, you can use generic JDBC connections in Apache Zeppelin. Go to [Generic JDBC Interpreter for Apache Zeppelin](../interpreter/jdbc.html). + +#### If you are in multi-user environment + * You can set permissions for your notebooks and secure data resource in multi-user environment. Go to **More** -> **Security** section. + ## Apache Zeppelin Configuration You can configure Apache Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority. diff --git a/docs/install/upgrade.md b/docs/install/upgrade.md index 21d44c2f0d9..a831eb98d4e 100644 --- a/docs/install/upgrade.md +++ b/docs/install/upgrade.md @@ -19,12 +19,14 @@ limitations under the License. --> {% include JB/setup %} -## Manual upgrade procedure for Zeppelin +# Manual upgrade procedure for Zeppelin + +
    Basically, newer version of Zeppelin works with previous version notebook directory and configurations. So, copying `notebook` and `conf` directory should be enough. -### Instructions +## Instructions 1. Stop Zeppelin ``` @@ -33,7 +35,7 @@ So, copying `notebook` and `conf` directory should be enough. 1. Copy your `notebook` and `conf` directory into a backup directory -1. Download newer version of Zeppelin and Install. See [Install page](./install.html) +1. Download newer version of Zeppelin and Install. See [Install page](./install.html#installation). 1. Copy backup `notebook` and `conf` directory into newer version of Zeppelin `notebook` and `conf` directory @@ -41,4 +43,4 @@ So, copying `notebook` and `conf` directory should be enough. ``` bin/zeppelin-daemon.sh start - ``` + ``` \ No newline at end of file diff --git a/docs/install/virtual_machine.md b/docs/install/virtual_machine.md index 7849ff171f0..7360183f5f9 100644 --- a/docs/install/virtual_machine.md +++ b/docs/install/virtual_machine.md @@ -19,8 +19,11 @@ limitations under the License. --> {% include JB/setup %} +# Vagrant Virtual Machine for Apache Zeppelin -## Vagrant Virtual Machine for Apache Zeppelin +
    + +## Overview Apache Zeppelin distribution includes a scripts directory @@ -30,11 +33,11 @@ This script creates a virtual machine that launches a repeatable, known set of c For PySpark users, this script includes several helpful [Python Libraries](#python-extras). For SparkR users, this script includes several helpful [R Libraries](#r-extras). -####Installing the required components to launch a virtual machine. +### Prerequisites This script requires three applications, [Ansible](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip "Ansible"), [Vagrant](http://www.vagrantup.com "Vagrant") and [Virtual Box](https://www.virtualbox.org/ "Virtual Box"). All of these applications are freely available as Open Source projects and extremely easy to set up on most operating systems. -### Create a Zeppelin Ready VM in 4 Steps (5 on Windows) +## Create a Zeppelin Ready VM If you are running Windows and don't yet have python installed, [install Python 2.7.x](https://www.python.org/downloads/release/python-2710/) first. @@ -60,9 +63,15 @@ curl -fsSL https://raw.githubusercontent.com/NFLabs/z-manager/master/zeppelin-in ``` -### Building Zeppelin +## Building Zeppelin + +You can now -You can now `git clone git://git.apache.org/zeppelin.git` into a directory on your host machine, or directly in your virtual machine. +``` +git clone git://git.apache.org/zeppelin.git +``` + +into a directory on your host machine, or directly in your virtual machine. Cloning Zeppelin into the `/scripts/vagrant/zeppelin-dev` directory from the host, will allow the directory to be shared between your host and the guest machine. @@ -74,14 +83,13 @@ By default, Vagrant will share your project directory (the directory with the Va `cd /vagrant/zeppelin` -### What's in this VM? +## What's in this VM? Running the following commands in the guest machine should display these expected versions: `node --version` should report *v0.12.7* `mvn --version` should report *Apache Maven 3.3.3* and *Java version: 1.7.0_85* - The virtual machine consists of: - Ubuntu Server 14.04 LTS @@ -96,7 +104,7 @@ The virtual machine consists of: - Python addons: pip, matplotlib, scipy, numpy, pandas - [R](https://www.r-project.org/) and R Packages required to run the R Interpreter and the related R tutorial notebook, including: Knitr, devtools, repr, rCharts, ggplot2, googleVis, mplot, htmltools, base64enc, data.table -### How to build & run Zeppelin +## How to build & run Zeppelin This assumes you've already cloned the project either on the host machine in the zeppelin-dev directory (to be shared with the guest machine) or cloned directly into a directory while running inside the guest machine. The following build steps will also include Python and R support via PySpark and SparkR: @@ -111,7 +119,7 @@ On your host machine browse to `http://localhost:8080/` If you [turned off port forwarding](#tweaking-the-virtual-machine) in the `Vagrantfile` browse to `http://192.168.51.52:8080` -### Tweaking the Virtual Machine +## Tweaking the Virtual Machine If you plan to run this virtual machine along side other Vagrant images, you may wish to bind the virtual machine to a specific IP address, and not use port fowarding from your local host. @@ -125,7 +133,7 @@ config.vm.network "private_network", ip: "192.168.51.52" `vagrant halt` followed by `vagrant up` will restart the guest machine bound to the IP address of `192.168.51.52`. This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources. You may wish to launch nodes in virtual machines with IP addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc.. - +## Extras ### Python Extras With Zeppelin running, **Numpy**, **SciPy**, **Pandas** and **Matplotlib** will be available. Create a pyspark notebook, and try the below code. diff --git a/docs/interpreter/alluxio.md b/docs/interpreter/alluxio.md index 332dd0d24b2..c8ecf93d281 100644 --- a/docs/interpreter/alluxio.md +++ b/docs/interpreter/alluxio.md @@ -6,7 +6,11 @@ group: manual --- {% include JB/setup %} -## Alluxio Interpreter for Apache Zeppelin +# Alluxio Interpreter for Apache Zeppelin + +
    + +## Overview [Alluxio](http://alluxio.org/) is a memory-centric distributed storage system enabling reliable data sharing at memory-speed across cluster frameworks. ## Configuration diff --git a/docs/interpreter/cassandra.md b/docs/interpreter/cassandra.md index 2091666761c..33cff199b05 100644 --- a/docs/interpreter/cassandra.md +++ b/docs/interpreter/cassandra.md @@ -6,7 +6,9 @@ group: manual --- {% include JB/setup %} -## Cassandra CQL Interpreter for Apache Zeppelin +# Cassandra CQL Interpreter for Apache Zeppelin + +
    diff --git a/docs/interpreter/elasticsearch.md b/docs/interpreter/elasticsearch.md index 70af3c09846..4721bcda3f2 100644 --- a/docs/interpreter/elasticsearch.md +++ b/docs/interpreter/elasticsearch.md @@ -6,7 +6,11 @@ group: manual --- {% include JB/setup %} -## Elasticsearch Interpreter for Apache Zeppelin +# Elasticsearch Interpreter for Apache Zeppelin + +
    + +## Overview [Elasticsearch](https://www.elastic.co/products/elasticsearch) is a highly scalable open-source full-text search and analytics engine. It allows you to store, search, and analyze big volumes of data quickly and in near real time. It is generally used as the underlying engine/technology that powers applications that have complex search features and requirements. ## Configuration diff --git a/docs/interpreter/flink.md b/docs/interpreter/flink.md index 9d2f0b05a9f..a678480b59f 100644 --- a/docs/interpreter/flink.md +++ b/docs/interpreter/flink.md @@ -6,7 +6,11 @@ group: manual --- {% include JB/setup %} -## Flink interpreter for Apache Zeppelin +# Flink interpreter for Apache Zeppelin + +
    + +## Overview [Apache Flink](https://flink.apache.org) is an open source platform for distributed stream and batch data processing. Flink’s core is a streaming dataflow engine that provides data distribution, communication, and fault tolerance for distributed computations over data streams. Flink also builds batch processing on top of the streaming engine, overlaying native iteration support, managed memory, and program optimization. ## How to start local Flink cluster, to test the interpreter diff --git a/docs/interpreter/geode.md b/docs/interpreter/geode.md index 53a912e3ac6..84a026efff5 100644 --- a/docs/interpreter/geode.md +++ b/docs/interpreter/geode.md @@ -6,7 +6,11 @@ group: manual --- {% include JB/setup %} -## Geode/Gemfire OQL Interpreter for Apache Zeppelin +# Geode/Gemfire OQL Interpreter for Apache Zeppelin + +
    + +## Overview
    @@ -33,7 +37,7 @@ This interpreter supports the [Geode](http://geode.incubator.apache.org/) [Objec This [Video Tutorial](https://www.youtube.com/watch?v=zvzzA9GXu3Q) illustrates some of the features provided by the `Geode Interpreter`. -### Create Interpreter +## Create Interpreter By default Zeppelin creates one `Geode/OQL` instance. You can remove it or create more instances. Multiple Geode instances can be created, each configured to the same or different backend Geode cluster. But over time a `Notebook` can have only one Geode interpreter instance `bound`. That means you _cannot_ connect to different Geode clusters in the same `Notebook`. This is a known Zeppelin limitation. @@ -42,10 +46,10 @@ To create new Geode instance open the `Interpreter` section and click the `+Crea > Note: The `Name` of the instance is used only to distinguish the instances while binding them to the `Notebook`. The `Name` is irrelevant inside the `Notebook`. In the `Notebook` you must use `%geode.oql` tag. -### Bind to Notebook +## Bind to Notebook In the `Notebook` click on the `settings` icon in the top right corner. The select/deselect the interpreters to be bound with the `Notebook`. -### Configuration +## Configuration You can modify the configuration of the Geode from the `Interpreter` section. The Geode interpreter expresses the following properties:
    Name
    @@ -71,12 +75,12 @@ You can modify the configuration of the Geode from the `Interpreter` section. T
    -### How to use +## How to use > *Tip 1: Use (CTRL + .) for OQL auto-completion.* > *Tip 2: Always start the paragraphs with the full `%geode.oql` prefix tag! The short notation: `%geode` would still be able run the OQL queries but the syntax highlighting and the auto-completions will be disabled.* -#### Create / Destroy Regions +### Create / Destroy Regions The OQL specification does not support [Geode Regions](https://cwiki.apache.org/confluence/display/GEODE/Index#Index-MainConceptsandComponents) mutation operations. To `create`/`destroy` regions one should use the [GFSH](http://geode-docs.cfapps.io/docs/tools_modules/gfsh/chapter_overview.html) shell tool instead. In the following it is assumed that the GFSH is colocated with Zeppelin server. ```bash @@ -97,7 +101,7 @@ EOF Above snippet re-creates two regions: `regionEmployee` and `regionCompany`. Note that you have to explicitly specify the locator host and port. The values should match those you have used in the Geode Interpreter configuration. Comprehensive list of [GFSH Commands by Functional Area](http://geode-docs.cfapps.io/docs/tools_modules/gfsh/gfsh_quick_reference.html). -#### Basic OQL +### Basic OQL ```sql %geode.oql SELECT count(*) FROM /regionEmployee @@ -136,7 +140,7 @@ SELECT e.key, e.value FROM /regionEmployee.entrySet e > Note: You can have multiple queries in the same paragraph but only the result from the first is displayed. [[1](https://issues.apache.org/jira/browse/ZEPPELIN-178)], [[2](https://issues.apache.org/jira/browse/ZEPPELIN-212)]. -#### GFSH Commands From The Shell +### GFSH Commands From The Shell Use the Shell Interpreter (`%sh`) to run OQL commands form the command line: ```bash @@ -145,7 +149,7 @@ source /etc/geode/conf/geode-env.sh gfsh -e "connect" -e "list members" ``` -#### Apply Zeppelin Dynamic Forms +### Apply Zeppelin Dynamic Forms You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your OQL queries. You can use both the `text input` and `select form` parameterization features ```sql @@ -153,7 +157,10 @@ You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your SELECT * FROM /regionEmployee e WHERE e.employeeId > ${Id} ``` -#### Geode REST API +### Auto-completion +The Geode Interpreter provides a basic auto-completion functionality. On `(Ctrl+.)` it list the most relevant suggestions in a pop-up window. + +## Geode REST API To list the defined regions you can use the [Geode REST API](http://geode-docs.cfapps.io/docs/geode_rest/chapter_overview.html): ``` @@ -182,6 +189,3 @@ http://phd1.localdomain:8484/gemfire-api/v1/ http-service-port=8484 start-dev-rest-api=true ``` - -### Auto-completion -The Geode Interpreter provides a basic auto-completion functionality. On `(Ctrl+.)` it list the most relevant suggestions in a pop-up window. diff --git a/docs/interpreter/hbase.md b/docs/interpreter/hbase.md index 2eaa91578f7..1aeb77bcade 100644 --- a/docs/interpreter/hbase.md +++ b/docs/interpreter/hbase.md @@ -6,16 +6,22 @@ group: manual --- {% include JB/setup %} -## HBase Shell Interpreter for Apache Zeppelin +# HBase Shell Interpreter for Apache Zeppelin + +
    + +## Overview [HBase Shell](http://hbase.apache.org/book.html#shell) is a JRuby IRB client for Apache HBase. This interpreter provides all capabilities of Apache HBase shell within Apache Zeppelin. The interpreter assumes that Apache HBase client software has been installed and it can connect to the Apache HBase cluster from the machine on where Apache Zeppelin is installed. -To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/book.html#quickstart) +To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/book.html#quickstart). ## HBase release supported By default, Zeppelin is built against HBase 1.0.x releases. To work with HBase 1.1.x releases, use the following build command: + ```bash # HBase 1.1.4 mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.1.4 -Dhbase.hadoop.version=2.6.0 ``` + To work with HBase 1.2.0+, use the following build command: ```bash @@ -94,4 +100,4 @@ And then to put data into that table put 'test', 'row1', 'cf:a', 'value1' ``` -For more information on all commands available, refer to [HBase shell commands](https://learnhbase.wordpress.com/2013/03/02/hbase-shell-commands/) +For more information on all commands available, refer to [HBase shell commands](https://learnhbase.wordpress.com/2013/03/02/hbase-shell-commands/). diff --git a/docs/interpreter/hdfs.md b/docs/interpreter/hdfs.md index 58d825dddb3..7cde31a6960 100644 --- a/docs/interpreter/hdfs.md +++ b/docs/interpreter/hdfs.md @@ -6,8 +6,11 @@ group: manual --- {% include JB/setup %} -## HDFS File System Interpreter for Apache Zeppelin +# HDFS File System Interpreter for Apache Zeppelin +
    + +## Overview [Hadoop File System](http://hadoop.apache.org/) is a distributed, fault tolerant file system part of the hadoop project and is often used as storage for distributed processing engines like [Hadoop MapReduce](http://hadoop.apache.org/) and [Apache Spark](http://spark.apache.org/) or underlying file systems like [Alluxio](http://www.alluxio.org/). ## Configuration @@ -44,13 +47,17 @@ It supports the basic shell file commands applied to HDFS, it currently only sup > **Tip :** Use ( Ctrl + . ) for autocompletion. -### Create Interpreter +## Create Interpreter In a notebook, to enable the **HDFS** interpreter, click the **Gear** icon and select **HDFS**. -#### WebHDFS REST API +## WebHDFS REST API You can confirm that you're able to access the WebHDFS API by running a curl command against the WebHDFS end point provided to the interpreter. Here is an example: + +```bash $> curl "http://localhost:50070/webhdfs/v1/?op=LISTSTATUS" +``` + diff --git a/docs/interpreter/hive.md b/docs/interpreter/hive.md index 2fc365c3502..a1fc4e1e618 100644 --- a/docs/interpreter/hive.md +++ b/docs/interpreter/hive.md @@ -6,8 +6,9 @@ group: manual --- {% include JB/setup %} -## Hive Interpreter for Apache Zeppelin -The [Apache Hive](https://hive.apache.org/) ™ data warehouse software facilitates querying and managing large datasets residing in distributed storage. Hive provides a mechanism to project structure onto this data and query the data using a SQL-like language called HiveQL. At the same time this language also allows traditional map/reduce programmers to plug in their custom mappers and reducers when it is inconvenient or inefficient to express this logic in HiveQL. +# Hive Interpreter for Apache Zeppelin + +
    ## Important Notice Hive Interpreter will be deprecated and merged into JDBC Interpreter. You can use Hive Interpreter by using JDBC Interpreter with same functionality. See the example below of settings and dependencies. @@ -52,7 +53,6 @@ Hive Interpreter will be deprecated and merged into JDBC Interpreter. You can us ----- ### Configuration @@ -115,6 +115,10 @@ Hive Interpreter will be deprecated and merged into JDBC Interpreter. You can us This interpreter provides multiple configuration with `${prefix}`. User can set a multiple connection properties by this prefix. It can be used like `%hive(${prefix})`. +## Overview + +The [Apache Hive](https://hive.apache.org/) ™ data warehouse software facilitates querying and managing large datasets residing in distributed storage. Hive provides a mechanism to project structure onto this data and query the data using a SQL-like language called HiveQL. At the same time this language also allows traditional map/reduce programmers to plug in their custom mappers and reducers when it is inconvenient or inefficient to express this logic in HiveQL. + ## How to use Basically, you can use diff --git a/docs/interpreter/ignite.md b/docs/interpreter/ignite.md index 6bc20abb5ed..8a25fd7ca79 100644 --- a/docs/interpreter/ignite.md +++ b/docs/interpreter/ignite.md @@ -6,16 +6,18 @@ group: manual --- {% include JB/setup %} -## Ignite Interpreter for Apache Zeppelin +# Ignite Interpreter for Apache Zeppelin -### Overview +
    + +## Overview [Apache Ignite](https://ignite.apache.org/) In-Memory Data Fabric is a high-performance, integrated and distributed in-memory platform for computing and transacting on large-scale data sets in real-time, orders of magnitude faster than possible with traditional disk-based or flash technologies. ![Apache Ignite](../assets/themes/zeppelin/img/docs-img/ignite-logo.png) You can use Zeppelin to retrieve distributed data from cache using Ignite SQL interpreter. Moreover, Ignite interpreter allows you to execute any Scala code in cases when SQL doesn't fit to your requirements. For example, you can populate data into your caches or execute distributed computations. -### Installing and Running Ignite example +## Installing and Running Ignite example In order to use Ignite interpreters, you may install Apache Ignite in some simple steps: 1. Download Ignite [source release](https://ignite.apache.org/download.html#sources) or [binary release](https://ignite.apache.org/download.html#binaries) whatever you want. But you must download Ignite as the same version of Zeppelin's. If it is not, you can't use scala code on Zeppelin. You can find ignite version in Zeppelin at the pom.xml which is placed under `path/to/your-Zeppelin/ignite/pom.xml` ( Of course, in Zeppelin source release ). Please check `ignite.version` .
    Currently, Zeppelin provides ignite only in Zeppelin source release. So, if you download Zeppelin binary release( `zeppelin-0.5.0-incubating-bin-spark-xxx-hadoop-xx` ), you can not use ignite interpreter on Zeppelin. We are planning to include ignite in a future binary release. @@ -31,7 +33,7 @@ In order to use Ignite interpreters, you may install Apache Ignite in some simpl $ nohup java -jar ``` -### Configuring Ignite Interpreter +## Configuring Ignite Interpreter At the "Interpreters" menu, you may edit Ignite interpreter or create new one. Zeppelin provides these properties for Ignite.
    @@ -69,14 +71,14 @@ At the "Interpreters" menu, you may edit Ignite interpreter or create new one. Z ![Configuration of Ignite Interpreter](../assets/themes/zeppelin/img/docs-img/ignite-interpreter-setting.png) -### Interpreter Binding for Zeppelin Notebook +## How to use After configuring Ignite interpreter, create your own notebook. Then you can bind interpreters like below image. ![Binding Interpreters](../assets/themes/zeppelin/img/docs-img/ignite-interpreter-binding.png) For more interpreter binding information see [here](http://zeppelin.apache.org/docs/manual/interpreters.html). -### How to use Ignite SQL interpreter +### Ignite SQL interpreter In order to execute SQL query, use ` %ignite.ignitesql ` prefix.
    Supposing you are running `org.apache.ignite.examples.streaming.wordcount.StreamWords`, then you can use "words" cache( Of course you have to specify this cache name to the Ignite interpreter setting section `ignite.jdbc.url` of Zeppelin ). For example, you can select top 10 words in the words cache using the following query diff --git a/docs/interpreter/jdbc.md b/docs/interpreter/jdbc.md index c4eef986d2f..830dd9789e4 100644 --- a/docs/interpreter/jdbc.md +++ b/docs/interpreter/jdbc.md @@ -7,7 +7,11 @@ group: manual {% include JB/setup %} -## Generic JDBC Interpreter for Apache Zeppelin +# Generic JDBC Interpreter for Apache Zeppelin + +
    + +## Overview This interpreter lets you create a JDBC connection to any data source, by now it has been tested with: @@ -16,16 +20,14 @@ This interpreter lets you create a JDBC connection to any data source, by now it * MariaDB * Redshift * Apache Hive -* Apache Drill - * Details on using [Drill JDBC Driver](https://drill.apache.org/docs/using-the-jdbc-driver) * Apache Phoenix -* Apache Tajo +* Apache Drill (Details on using [Drill JDBC Driver](https://drill.apache.org/docs/using-the-jdbc-driverde* Apache Tajo If someone else used another database please report how it works to improve functionality. -### Create Interpreter +## Create Interpreter -When create a interpreter by default use PostgreSQL with the next properties: +When you create a interpreter by default use PostgreSQL with the next properties:
    @@ -56,7 +58,7 @@ When create a interpreter by default use PostgreSQL with the next properties: It is not necessary to add driver jar to the classpath for PostgreSQL as it is included in Zeppelin. -#### Simple connection +### Simple connection Prior to creating the interpreter it is necessary to add maven coordinate or path of the JDBC driver to the Zeppelin classpath. To do this you must edit dependencies artifact(ex. `mysql:mysql-connector-java:5.1.38`) in interpreter menu as shown: @@ -95,7 +97,7 @@ To create the interpreter you need to specify connection parameters as shown in
    -#### Multiple connections +### Multiple connections JDBC interpreter also allows connections to multiple data sources. It is necessary to set a prefix for each connection to reference it in the paragraph in the form of `%jdbc(prefix)`. Before you create the interpreter it is necessary to add each driver's maven coordinates or JDBC driver's jar file path to the Zeppelin classpath. To do this you must edit the dependencies of JDBC interpreter in interpreter menu as following: @@ -151,10 +153,10 @@ You can add all the jars you need to make multiple connections into the same JDB -### Bind to Notebook +## Bind to Notebook In the `Notebook` click on the `settings` icon at the top-right corner. Use select/deselect to specify the interpreters to be used in the `Notebook`. -### More Properties +## More Properties You can modify the interpreter configuration in the `Interpreter` section. The most common properties are as follows, but you can specify other properties that need to be connected. @@ -197,9 +199,11 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
    -### Examples -#### Hive -##### Properties +## Examples + +### Hive + +#### Properties @@ -222,7 +226,8 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
    Namehive_password
    -##### Dependencies + +#### Dependencies @@ -237,8 +242,9 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
    Artifact
    -#### Phoenix -##### Properties + +### Phoenix +#### Properties @@ -261,7 +267,7 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
    Namephoenix_password
    -##### Dependencies +#### Dependencies @@ -272,8 +278,9 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
    Artifact
    -#### Tajo -##### Properties + +### Tajo +#### Properties @@ -288,7 +295,8 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
    Namejdbc:tajo://localhost:26002/default
    -##### Dependencies + +#### Dependencies @@ -300,9 +308,9 @@ To develop this functionality use this [method](http://docs.oracle.com/javase/7/
    Artifact
    -### How to use +## How to use -#### Reference in paragraph +### Reference in paragraph Start the paragraphs with the `%jdbc`, this will use the `default` prefix for connection. If you want to use other connection you should specify the prefix of it as follows `%jdbc(prefix)`: @@ -311,6 +319,7 @@ Start the paragraphs with the `%jdbc`, this will use the `default` prefix for co SELECT * FROM db_name; ``` + or ```sql @@ -319,7 +328,7 @@ SELECT * FROM db_name; ``` -#### Apply Zeppelin Dynamic Forms +### Apply Zeppelin Dynamic Forms You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parametrization features @@ -330,5 +339,5 @@ FROM demo.performers WHERE name='{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia}}' ``` -### Bugs & Contacts +## Bugs & Reporting If you find a bug for this interpreter, please create a [JIRA]( https://issues.apache.org/jira/browse/ZEPPELIN-382?jql=project%20%3D%20ZEPPELIN) ticket. diff --git a/docs/interpreter/lens.md b/docs/interpreter/lens.md index 0b4711bace5..b4bcda49bd7 100644 --- a/docs/interpreter/lens.md +++ b/docs/interpreter/lens.md @@ -6,14 +6,16 @@ group: manual --- {% include JB/setup %} -## Lens Interpreter for Apache Zeppelin +# Lens Interpreter for Apache Zeppelin -### Overview +
    + +## Overview [Apache Lens](https://lens.apache.org/) provides an Unified Analytics interface. Lens aims to cut the Data Analytics silos by providing a single view of data across multiple tiered data stores and optimal execution environment for the analytical query. It seamlessly integrates Hadoop with traditional data warehouses to appear like one. ![Apache Lens](../assets/themes/zeppelin/img/docs-img/lens-logo.png) -### Installing and Running Lens +## Installing and Running Lens In order to use Lens interpreters, you may install Apache Lens in some simple steps: 1. Download Lens for latest version from [the ASF](http://www.apache.org/dyn/closer.lua/lens/2.3-beta). Or the older release can be found [in the Archives](http://archive.apache.org/dist/lens/). @@ -24,7 +26,7 @@ In order to use Lens interpreters, you may install Apache Lens in some simple st ./bin/lens-ctl start (or stop) ``` -### Configuring Lens Interpreter +## Configuring Lens Interpreter At the "Interpreters" menu, you can edit Lens interpreter or create new one. Zeppelin provides these properties for Lens. @@ -163,7 +165,7 @@ query execute cube select customer_city_name, product_details.description, produ These are just examples that provided in advance by Lens. If you want to explore whole tutorials of Lens, see the [tutorial video](https://cwiki.apache.org/confluence/display/LENS/2015/07/13/20+Minute+video+demo+of+Apache+Lens+through+examples). -### Lens UI Service +## Lens UI Service Lens also provides web UI service. Once the server starts up, you can open the service on http://serverhost:19999/index.html and browse. You may also check the structure that you made and use query easily here. ![Lens UI Service](../assets/themes/zeppelin/img/docs-img/lens-ui-service.png) diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index 225cd817bee..ef7c8ce6876 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -6,8 +6,12 @@ group: manual --- {% include JB/setup %} -## Livy Interpreter for Apache Zeppelin -Livy is an open source REST interface for interacting with Spark from anywhere. It supports executing snippets of code or programs in a Spark context that runs locally or in YARN. +# Livy Interpreter for Apache Zeppelin + +
    + +## Overview +[Livy](http://livy.io/) is an open source REST interface for interacting with Spark from anywhere. It supports executing snippets of code or programs in a Spark context that runs locally or in YARN. * Interactive Scala, Python and R shells * Batch submissions in Scala, Java, Python @@ -16,13 +20,12 @@ Livy is an open source REST interface for interacting with Spark from anywhere. * Does not require any code change to your programs ### Requirements - Additional requirements for the Livy interpreter are: * Spark 1.3 or above. * Livy server. -### Configuration +## Configuration We added some common configurations for spark, and you can set any configuration you want. This link contains all spark configurations: http://spark.apache.org/docs/latest/configuration.html#available-properties. And instead of starting property with `spark.` it should be replaced with `livy.spark.`. @@ -101,8 +104,6 @@ Example: `spark.master` to `livy.spark.master`
    - - ## How to use Basically, you can use @@ -136,7 +137,7 @@ hello("livy") When Zeppelin server is running with authentication enabled, then this interpreter utilizes Livy’s user impersonation feature i.e. sends extra parameter for creating and running a session ("proxyUser": "${loggedInUser}"). This is particularly useful when multi users are sharing a Notebook server. -### Apply Zeppelin Dynamic Forms +## Apply Zeppelin Dynamic Forms You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html). You can use both the `text input` and `select form` parameterization features. ``` @@ -159,7 +160,7 @@ The session would have timed out, you may need to restart the interpreter. > Blacklisted configuration values in session config: spark.master -edit `conf/spark-blacklist.conf` file in livy server and comment out `#spark.master` line. +Edit `conf/spark-blacklist.conf` file in livy server and comment out `#spark.master` line. -if you choose to work on livy in `apps/spark/java` directory in https://github.com/cloudera/hue , -copy `spark-user-configurable-options.template` to `spark-user-configurable-options.conf` file in livy server and comment out `#spark.master` +If you choose to work on livy in `apps/spark/java` directory in [https://github.com/cloudera/hue](https://github.com/cloudera/hue), +copy `spark-user-configurable-options.template` to `spark-user-configurable-options.conf` file in livy server and comment out `#spark.master`. diff --git a/docs/interpreter/markdown.md b/docs/interpreter/markdown.md index 08b44f84f20..21184dcf762 100644 --- a/docs/interpreter/markdown.md +++ b/docs/interpreter/markdown.md @@ -6,9 +6,11 @@ group: manual --- {% include JB/setup %} -## Markdown Interpreter for Apache Zeppelin +# Markdown Interpreter for Apache Zeppelin -### Overview +
    + +## Overview [Markdown](http://daringfireball.net/projects/markdown/) is a plain text formatting syntax designed so that it can be converted to HTML. Zeppelin uses markdown4j. For more examples and extension support, please checkout [here](https://code.google.com/p/markdown4j/). In Zeppelin notebook, you can use ` %md ` in the beginning of a paragraph to invoke the Markdown interpreter and generate static html from Markdown plain text. @@ -17,7 +19,7 @@ In Zeppelin, Markdown interpreter is enabled by default. -### Example +## Example The following example demonstrates the basic usage of Markdown in a Zeppelin notebook. diff --git a/docs/interpreter/postgresql.md b/docs/interpreter/postgresql.md index 5985b188791..107fda1d809 100644 --- a/docs/interpreter/postgresql.md +++ b/docs/interpreter/postgresql.md @@ -6,7 +6,12 @@ group: manual --- {% include JB/setup %} +# PostgreSQL, Apache HAWQ (incubating) Interpreter for Apache Zeppelin + +
    + ## Important Notice + Postgresql Interpreter will be deprecated and merged into JDBC Interpreter. You can use Postgresql by using JDBC Interpreter with same functionality. See the example below of settings and dependencies. ### Properties @@ -44,10 +49,19 @@ Postgresql Interpreter will be deprecated and merged into JDBC Interpreter. You +--- ----- +## Overview -## PostgreSQL, HAWQ Interpreter for Apache Zeppelin +[zeppelin-view](https://www.youtube.com/watch?v=wqXXQhJ5Uk8) + +This interpreter seamlessly supports the following SQL data processing engines: + +* [PostgreSQL](http://www.postgresql.org/) - OSS, Object-relational database management system (ORDBMS) +* [pache HAWQ (incubating)](http://hawq.incubator.apache.org/) - Powerful open source SQL-On-Hadoop engine. +* [Greenplum](http://pivotal.io/big-data/pivotal-greenplum-database) - MPP database built on open source PostgreSQL. + +This [Video Tutorial](https://www.youtube.com/watch?v=wqXXQhJ5Uk8) illustrates some of the features provided by the `Postgresql Interpreter`. @@ -62,17 +76,7 @@ Postgresql Interpreter will be deprecated and merged into JDBC Interpreter. You
    -[zeppelin-view](https://www.youtube.com/watch?v=wqXXQhJ5Uk8) - -This interpreter seamlessly supports the following SQL data processing engines: - -* [PostgreSQL](http://www.postgresql.org/) - OSS, Object-relational database management system (ORDBMS) -* [Apache HAWQ](http://pivotal.io/big-data/pivotal-hawq) - Powerful [Open Source](https://wiki.apache.org/incubator/HAWQProposal) SQL-On-Hadoop engine. -* [Greenplum](http://pivotal.io/big-data/pivotal-greenplum-database) - MPP database built on open source PostgreSQL. - -This [Video Tutorial](https://www.youtube.com/watch?v=wqXXQhJ5Uk8) illustrates some of the features provided by the `Postgresql Interpreter`. - -### Create Interpreter +## Create Interpreter By default Zeppelin creates one `PSQL` instance. You can remove it or create new instances. Multiple PSQL instances can be created, each configured to the same or different backend databases. But over time a `Notebook` can have only one PSQL interpreter instance `bound`. That means you _cannot_ connect to different databases in the same `Notebook`. This is a known Zeppelin limitation. @@ -81,10 +85,10 @@ To create new PSQL instance open the `Interpreter` section and click the `+Creat > Note: The `Name` of the instance is used only to distinct the instances while binding them to the `Notebook`. The `Name` is irrelevant inside the `Notebook`. In the `Notebook` you must use `%psql.sql` tag. -### Bind to Notebook +## Bind to Notebook In the `Notebook` click on the `settings` icon in the top right corner. The select/deselect the interpreters to be bound with the `Notebook`. -### Configuration +## Configuration You can modify the configuration of the PSQL from the `Interpreter` section. The PSQL interpreter expenses the following properties: @@ -120,12 +124,12 @@ You can modify the configuration of the PSQL from the `Interpreter` section. Th
    -### How to use +## How to use ``` Tip: Use (CTRL + .) for SQL auto-completion. ``` -#### DDL and SQL commands +### DDL and SQL commands Start the paragraphs with the full `%psql.sql` prefix tag! The short notation: `%psql` would still be able run the queries but the syntax highlighting and the auto-completions will be disabled. You can use the standard CREATE / DROP / INSERT commands to create or modify the data model: @@ -154,7 +158,7 @@ select count(*) from mytable; select * from mytable; ``` -#### PSQL command line tools +### PSQL command line tools Use the Shell Interpreter (`%sh`) to access the command line [PSQL](http://www.postgresql.org/docs/9.4/static/app-psql.html) interactively: ```bash @@ -179,7 +183,7 @@ This will produce output like this: retail_demo | gpadmin ``` -#### Apply Zeppelin Dynamic Forms +### Apply Zeppelin Dynamic Forms You can leverage [Zeppelin Dynamic Form](../manual/dynamicform.html) inside your queries. You can use both the `text input` and `select form` parametrization features ```sql @@ -191,7 +195,7 @@ ORDER BY count ${order=DESC,DESC|ASC} LIMIT ${limit=10}; ``` -#### Example HAWQ PXF/HDFS Tables +### Example HAWQ PXF/HDFS Tables Create HAWQ external table that read data from tab-separated-value data in HDFS. ```sql @@ -209,5 +213,5 @@ And retrieve content select * from retail_demo.payment_methods_pxf ``` -### Auto-completion +## Auto-completion The PSQL Interpreter provides a basic auto-completion functionality. On `(Ctrl+.)` it list the most relevant suggestions in a pop-up window. In addition to the SQL keyword the interpreter provides suggestions for the Schema, Table, Column names as well. diff --git a/docs/interpreter/python.md b/docs/interpreter/python.md index b34b0898ffc..c2a38f54433 100644 --- a/docs/interpreter/python.md +++ b/docs/interpreter/python.md @@ -6,7 +6,9 @@ group: manual --- {% include JB/setup %} -## Python 2 & 3 Interpreter for Apache Zeppelin +# Python 2 & 3 Interpreter for Apache Zeppelin + +
    ## Configuration @@ -63,8 +65,6 @@ print (z.select("f1",[("o1","1"),("o2","2")],"2")) print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"]))) ``` - - ## Zeppelin features not fully supported by the Python Interpreter * Interrupt a paragraph execution (`cancel()` method) is currently only supported in Linux and MacOs. If interpreter runs in another operating system (for instance MS Windows) , interrupt a paragraph will close the whole interpreter. A JIRA ticket ([ZEPPELIN-893](https://issues.apache.org/jira/browse/ZEPPELIN-893)) is opened to implement this feature in a next release of the interpreter. @@ -90,8 +90,7 @@ z.show function can take optional parameters to adapt graph width and height z.show(plt, width='50px') z.show(plt, height='150px') ``` - -[![pythonmatplotlib](../interpreter/screenshots/pythonMatplotlib.png)](/docs/interpreter/screenshots/pythonMatplotlib.png) + ## Pandas integration @@ -105,7 +104,6 @@ rates = pd.read_csv("bank.csv", sep=";") z.show(rates) ``` - ## Technical description -For in-depth technical details on current implementation plese reffer [python/README.md](https://github.com/apache/zeppelin/blob/master/python/README.md) +For in-depth technical details on current implementation plese reffer [python/README.md](https://github.com/apache/zeppelin/blob/master/python/README.md). diff --git a/docs/interpreter/r.md b/docs/interpreter/r.md index c5ed98b8da9..dce8bd1dcc9 100644 --- a/docs/interpreter/r.md +++ b/docs/interpreter/r.md @@ -6,78 +6,99 @@ group: manual --- {% include JB/setup %} -## R Interpreter +# R Interpreter for Apache Zeppelin -This is a the Apache Zeppelin project, with the addition of support for the R programming language and R-spark integration. +
    -### Requirements +## Overview -Additional requirements for the R interpreter are: +[R](https://www.r-project.org) is a free software environment for statistical computing and graphics. - * R 3.1 or later (earlier versions may work, but have not been tested) - * The `evaluate` R package. +To run R code and visualize plots in Apache Zeppelin, you will need R on your master node (or your dev laptop). -For full R support, you will also need the following R packages: ++ For Centos: `yum install R R-devel libcurl-devel openssl-devel` ++ For Ubuntu: `apt-get install r-base` - * `knitr` - * `repr` -- available with `devtools::install_github("IRkernel/repr")` - * `htmltools` -- required for some interactive plotting - * `base64enc` -- required to view R base plots +Validate your installation with a simple R command: -### Configuration +``` +R -e "print(1+1)" +``` -To run Zeppelin with the R Interpreter, the SPARK_HOME environment variable must be set. The best way to do this is by editing `conf/zeppelin-env.sh`. +To enjoy plots, install additional libraries with: +``` ++ devtools with `R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')"` ++ knitr with `R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org')"` ++ ggplot2 with `R -e "install.packages('ggplot2', repos = 'http://cran.us.r-project.org')"` ++ Other vizualisation librairies: `R -e "install.packages(c('devtools','mplot', 'googleVis'), repos = 'http://cran.us.r-project.org'); require(devtools); install_github('ramnathv/rCharts')"` +``` + +We recommend you to also install the following optional R libraries for happy data analytics: + ++ glmnet ++ pROC ++ data.table ++ caret ++ sqldf ++ wordcloud + +## Configuration + +To run Zeppelin with the R Interpreter, the `SPARK_HOME` environment variable must be set. The best way to do this is by editing `conf/zeppelin-env.sh`. If it is not set, the R Interpreter will not be able to interface with Spark. -You should also copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`. That will ensure that Zeppelin sees the R Interpreter the first time it starts up. +You should also copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`. That will ensure that Zeppelin sees the R Interpreter the first time it starts up. -### Using the R Interpreter +## Using the R Interpreter By default, the R Interpreter appears as two Zeppelin Interpreters, `%r` and `%knitr`. `%r` will behave like an ordinary REPL. You can execute commands as in the CLI. -[![2+2](screenshots/repl2plus2.png)](screenshots/repl2plus2.png) + R base plotting is fully supported -[![replhist](screenshots/replhist.png)](screenshots/replhist.png) + If you return a data.frame, Zeppelin will attempt to display it using Zeppelin's built-in visualizations. -[![replhist](screenshots/replhead.png)](screenshots/replhead.png) + `%knitr` interfaces directly against `knitr`, with chunk options on the first line: -[![knitgeo](screenshots/knitgeo.png)](screenshots/knitgeo.png) -[![knitstock](screenshots/knitstock.png)](screenshots/knitstock.png) -[![knitmotion](screenshots/knitmotion.png)](screenshots/knitmotion.png) + + + + + The two interpreters share the same environment. If you define a variable from `%r`, it will be within-scope if you then make a call using `knitr`. -### Using SparkR & Moving Between Languages +## Using SparkR & Moving Between Languages If `SPARK_HOME` is set, the `SparkR` package will be loaded automatically: -[![sparkrfaithful](screenshots/sparkrfaithful.png)](screenshots/sparkrfaithful.png) + The Spark Context and SQL Context are created and injected into the local environment automatically as `sc` and `sql`. The same context are shared with the `%spark`, `%sql` and `%pyspark` interpreters: -[![backtoscala](screenshots/backtoscala.png)](screenshots/backtoscala.png) + You can also make an ordinary R variable accessible in scala and Python: -[![varr1](screenshots/varr1.png)](screenshots/varr1.png) + And vice versa: -[![varscala](screenshots/varscala.png)](screenshots/varscala.png) -[![varr2](screenshots/varr2.png)](screenshots/varr2.png) + + + -### Caveats & Troubleshooting +## Caveats & Troubleshooting * Almost all issues with the R interpreter turned out to be caused by an incorrectly set `SPARK_HOME`. The R interpreter must load a version of the `SparkR` package that matches the running version of Spark, and it does this by searching `SPARK_HOME`. If Zeppelin isn't configured to interface with Spark in `SPARK_HOME`, the R interpreter will not be able to connect to Spark. @@ -98,40 +119,3 @@ And vice versa: * Error `unable to start device X11` with the repl interpreter. Check your shell login scripts to see if they are adjusting the `DISPLAY` environment variable. This is common on some operating systems as a workaround for ssh issues, but can interfere with R plotting. * akka Library Version or `TTransport` errors. This can happen if you try to run Zeppelin with a SPARK_HOME that has a version of Spark other than the one specified with `-Pspark-1.x` when Zeppelin was compiled. - - - - - -## R Interpreter for Apache Zeppelin - -[R](https://www.r-project.org) is a free software environment for statistical computing and graphics. - -To run R code and visualize plots in Apache Zeppelin, you will need R on your master node (or your dev laptop). - -+ For Centos: `yum install R R-devel libcurl-devel openssl-devel` -+ For Ubuntu: `apt-get install r-base` - -Validate your installation with a simple R command: - -``` -R -e "print(1+1)" -``` - -To enjoy plots, install additional libraries with: - -``` -+ devtools with `R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')"` -+ knitr with `R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org')"` -+ ggplot2 with `R -e "install.packages('ggplot2', repos = 'http://cran.us.r-project.org')"` -+ Other vizualisation librairies: `R -e "install.packages(c('devtools','mplot', 'googleVis'), repos = 'http://cran.us.r-project.org'); require(devtools); install_github('ramnathv/rCharts')"` -``` - -We recommend you to also install the following optional R libraries for happy data analytics: - -+ glmnet -+ pROC -+ data.table -+ caret -+ sqldf -+ wordcloud diff --git a/docs/interpreter/scalding.md b/docs/interpreter/scalding.md index ec5608bf3b3..e8774df67fa 100644 --- a/docs/interpreter/scalding.md +++ b/docs/interpreter/scalding.md @@ -6,17 +6,20 @@ group: manual --- {% include JB/setup %} -## Scalding Interpreter for Apache Zeppelin +# Scalding Interpreter for Apache Zeppelin + +
    + [Scalding](https://github.com/twitter/scalding) is an open source Scala library for writing MapReduce jobs. -### Building the Scalding Interpreter +## Building the Scalding Interpreter You have to first build the Scalding interpreter by enable the **scalding** profile as follows: ``` mvn clean package -Pscalding -DskipTests ``` -### Enabling the Scalding Interpreter +## Enabling the Scalding Interpreter In a notebook, to enable the **Scalding** interpreter, click on the **Gear** icon,select **Scalding**, and hit **Save**.
    @@ -27,7 +30,7 @@ In a notebook, to enable the **Scalding** interpreter, click on the **Gear** ico
    -### Configuring the Interpreter +## Configuring the Interpreter Scalding interpreter runs in two modes: @@ -65,9 +68,9 @@ For reducer estimation, you need to add something like: If you want to control the maximum number of open interpreters, you have to select "scoped" interpreter for note option and set max.open.instances argument. -### Testing the Interpreter +## Testing the Interpreter -#### Local mode +### Local mode In example, by using the [Alice in Wonderland](https://gist.github.com/johnynek/a47699caa62f4f38a3e2) tutorial, we will count words (of course!), and plot a graph of the top 10 words in the book. @@ -111,7 +114,7 @@ If you click on the icon for the pie chart, you should be able to see a chart li ![Scalding - Pie - Chart](../assets/themes/zeppelin/img/docs-img/scalding-pie.png) -#### HDFS mode +### HDFS mode **Test mode** @@ -146,7 +149,7 @@ a.toList This command should create a map reduce job. -### Future Work +## Future Work * Better user feedback (hadoop url, progress updates) * Ability to cancel jobs * Ability to dynamically load jars without restarting the interpreter diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index df5e83176f2..a183033bf7c 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -7,8 +7,14 @@ group: manual {% include JB/setup %} -## Spark Interpreter for Apache Zeppelin -[Apache Spark](http://spark.apache.org) is supported in Zeppelin with +# Spark Interpreter for Apache Zeppelin + +
    + +## Overview +[Apache Spark](http://spark.apache.org) is a fast and general-purpose cluster computing system. +It provides high-level APIs in Java, Scala, Python and R, and an optimized engine that supports general execution graphs +Apache Spark is supported in Zeppelin with Spark Interpreter group, which consists of five interpreters.
    @@ -200,13 +206,13 @@ Here are few examples: * SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh - export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg" + export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg" * SPARK_HOME/conf/spark-defaults.conf - spark.jars /path/mylib1.jar,/path/mylib2.jar - spark.jars.packages com.databricks:spark-csv_2.10:1.2.0 - spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip + spark.jars /path/mylib1.jar,/path/mylib2.jar + spark.jars.packages com.databricks:spark-csv_2.10:1.2.0 + spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip ### 3. Dynamic Dependency Loading via %dep interpreter > Note: `%dep` interpreter is deprecated since v0.6.0. @@ -344,7 +350,7 @@ select * from ${table=defaultTableName} where text like '%${search}%' To learn more about dynamic form, checkout [Dynamic Form](../manual/dynamicform.html). -### Interpreter setting option. +## Interpreter setting option Interpreter setting can choose one of 'shared', 'scoped', 'isolated' option. Spark interpreter creates separate scala compiler per each notebook but share a single SparkContext in 'scoped' mode (experimental). It creates separate SparkContext per each notebook in 'isolated' mode. @@ -354,7 +360,7 @@ Logical setup with Zeppelin, Kerberos Key Distribution Center (KDC), and Spark o -####Configuration Setup +### Configuration Setup 1. On the server that Zeppelin is installed, install Kerberos client modules and configuration, krb5.conf. This is to make the server communicate with KDC. diff --git a/docs/manual/dynamicform.md b/docs/manual/dynamicform.md index 6594767efec..b554fec1127 100644 --- a/docs/manual/dynamicform.md +++ b/docs/manual/dynamicform.md @@ -19,16 +19,18 @@ limitations under the License. --> {% include JB/setup %} -## Dynamic Form +# Dynamic Form -Zeppelin dynamically creates input forms. Depending on language backend, there're two different ways to create dynamic form. +
    + +Apache Zeppelin dynamically creates input forms. Depending on language backend, there're two different ways to create dynamic form. Custom language backend can select which type of form creation it wants to use. -### Using form Templates +## Using form Templates This mode creates form using simple template language. It's simple and easy to use. For example Markdown, Shell, SparkSql language backend uses it. -#### Text input form +### Text input form To create text input form, use `${formName}` templates. @@ -42,7 +44,7 @@ Also you can provide default value, using `${formName=defaultValue}`. -#### Select form +### Select form To create select form, use `${formName=defaultValue,option1|option2...}` @@ -54,7 +56,7 @@ Also you can separate option's display name and value, using `${formName=default -#### Checkbox form +### Checkbox form For multi-selection, you can create a checkbox form using `${checkbox:formName=defaultValue1|defaultValue2...,option1|option2...}`. The variable will be substituted by a comma-separated string based on the selected items. For example: @@ -64,13 +66,13 @@ Besides, you can specify the delimiter using `${checkbox(delimiter):formName=... -### Creates Programmatically +## Creates Programmatically Some language backend uses programmatic way to create form. For example [ZeppelinContext](../interpreter/spark.html#zeppelincontext) provides form creation API Here're some examples. -####Text input form +### Text input form
    @@ -91,7 +93,7 @@ print("Hello "+z.input("name"))
    -####Text input form with default value +### Text input form with default value
    @@ -112,7 +114,7 @@ print("Hello "+z.input("name", "sun"))
    -####Select form +### Select form
    diff --git a/docs/manual/dynamicinterpreterload.md b/docs/manual/dynamicinterpreterload.md index 0794314b53d..42dd74df61f 100644 --- a/docs/manual/dynamicinterpreterload.md +++ b/docs/manual/dynamicinterpreterload.md @@ -19,12 +19,13 @@ limitations under the License. --> {% include JB/setup %} -## Dynamic Interpreter Loading using REST API +# Dynamic Interpreter Loading using REST API + +
    Zeppelin provides pluggable interpreter architecture which results in a wide and variety of the supported backend system. In this section, we will introduce **Dynamic interpreter loading** using **REST API**. This concept actually comes from [Zeppelin Helium Proposal](https://cwiki.apache.org/confluence/display/ZEPPELIN/Helium+proposal). Before we start, if you are not familiar with the concept of **Zeppelin interpreter**, you can check out [Overview of Zeppelin interpreter](../manual/interpreters.html) first. -
    ## Overview In the past, Zeppelin was loading interpreter binaries from `/interpreter/[interpreter_name]` directory. They were configured by `zeppelin.interpreters` property in `conf/zeppelin-site.xml` or `ZEPPELIN_INTERPRETERS` env variables in `conf/zeppelin-env.sh`. They were loaded on Zeppelin server startup and stayed alive until the server was stopped. In order to simplify using 3rd party interpreters, we changed this way to **dynamically** load interpreters from **Maven Repository** using **REST API**. Hopefully, the picture below will help you to understand the process. @@ -32,7 +33,7 @@ In order to simplify using 3rd party interpreters, we changed this way to **dyna ## Load & Unload Interpreters Using REST API -### 1. Load +### Load You can **load** interpreters located in Maven repository using REST API, like this: ( Maybe, you are unfamiliar with `[interpreter_group_name]` or `[interpreter_name]`. If so, please checkout [Interpreters in Zeppelin](../manual/interpreter.html) again. ) @@ -69,21 +70,21 @@ http://127.0.0.1:8080/api/interpreter/load/md/markdown The meaning of each parameters is: 1. **Artifact** - - groupId: org.apache.zeppelin - - artifactId: zeppelin-markdown - - version: 0.6.0-SNAPSHOT + - groupId: org.apache.zeppelin + - artifactId: zeppelin-markdown + - version: 0.6.0-SNAPSHOT 2. **Class Name** - - Package Name: org.apache.zeppelin - - Interpreter Class Name: markdown.Markdown + - Package Name: org.apache.zeppelin + - Interpreter Class Name: markdown.Markdown 3. **Repository ( optional )** - - Url: http://dl.bintray.com/spark-packages/maven - - Snapshot: false + - Url: http://dl.bintray.com/spark-packages/maven + - Snapshot: false > Please note: The interpreters you downloaded need to be **reload**, when your Zeppelin server is down. -### 2. Unload +### Unload If you want to **unload** the interpreters using REST API, ``` @@ -95,7 +96,7 @@ In this case, the Restful method will be **DELETE**. ## What is the next step after Loading ? ### Q1. Where is the location of interpreters you downloaded ? - + Actually, the answer about this question is in the above picture. Once the REST API is called, the `.jar` files of interpreters you get are saved under `ZEPPELIN_HOME/local-repo` first. Then, they will be copied to `ZEPPELIN_HOME/interpreter` directory. So, please checkout your `ZEPPELIN_HOME/interpreter`. ### Q2. Then, how can I use this interpreter ? diff --git a/docs/manual/interpreterinstallation.md b/docs/manual/interpreterinstallation.md index d522e620e53..b940f597b76 100644 --- a/docs/manual/interpreterinstallation.md +++ b/docs/manual/interpreterinstallation.md @@ -21,6 +21,8 @@ limitations under the License. # Interpreter Installation +
    + Apache Zeppelin provides **Interpreter Installation** mechanism for whom downloaded Zeppelin `netinst` binary package, or just want to install another 3rd party interpreters. ## Community managed interpreters diff --git a/docs/manual/interpreters.md b/docs/manual/interpreters.md index 488d5e9f28c..a21d34e81a7 100644 --- a/docs/manual/interpreters.md +++ b/docs/manual/interpreters.md @@ -19,7 +19,12 @@ limitations under the License. --> {% include JB/setup %} -## Interpreters in Zeppelin +# Interpreters in Apache Zeppelin + +
    + +## Overview + In this section, we will explain about the role of interpreters, interpreters group and interpreter settings in Zeppelin. The concept of Zeppelin interpreter allows any language/data-processing-backend to be plugged into Zeppelin. Currently, Zeppelin supports many interpreters such as Scala ( with Apache Spark ), Python ( with Apache Spark ), SparkSQL, JDBC, Markdown, Shell and so on. @@ -29,12 +34,12 @@ Zeppelin Interpreter is a plug-in which enables Zeppelin users to use a specific When you click the ```+Create``` button in the interpreter page, the interpreter drop-down list box will show all the available interpreters on your server. - + -## What is Zeppelin Interpreter Setting? +## What is interpreter setting? Zeppelin interpreter setting is the configuration of a given interpreter on Zeppelin server. For example, the properties are required for hive JDBC interpreter to connect to the Hive server. - + Properties are exported as environment variable when property name is consisted of upper characters, numbers and underscore ([A-Z_0-9]). Otherwise set properties as JVM property. @@ -44,14 +49,15 @@ Each notebook can be bound to multiple Interpreter Settings using setting icon o -## What is Zeppelin Interpreter Group? +## What is interpreter group? Every Interpreter is belonged to an **Interpreter Group**. Interpreter Group is a unit of start/stop interpreter. By default, every interpreter is belonged to a single group, but the group might contain more interpreters. For example, Spark interpreter group is including Spark support, pySpark, SparkSQL and the dependency loader. Technically, Zeppelin interpreters from the same group are running in the same JVM. For more information about this, please checkout [here](../development/writingzeppelininterpreter.html). Each interpreters is belonged to a single group and registered together. All of their properties are listed in the interpreter setting like below image. - + + ## Interpreter binding mode @@ -62,7 +68,7 @@ In 'shared' mode, every notebook bound to the Interpreter Setting will share the -## Connecting to the Existing Remote Interpreter +## Connecting to the existing remote interpreter Zeppelin users can start interpreter thread embedded in their service. This will provide flexibility to user to start interpreter on remote host. To start interpreter along with your service you have to create an instance of ``RemoteInterpreterServer`` and start it as follows: @@ -75,4 +81,4 @@ interpreter.start() The above code will start interpreter thread inside your process. Once the interpreter is started you can configure zeppelin to connect to RemoteInterpreter by checking **Connect to existing process** checkbox and then provide **Host** and **Port** on which interpreter porocess is listening as shown in the image below: - + diff --git a/docs/manual/notebookashomepage.md b/docs/manual/notebookashomepage.md index 48f06a6df45..957e61a4d52 100644 --- a/docs/manual/notebookashomepage.md +++ b/docs/manual/notebookashomepage.md @@ -19,91 +19,84 @@ limitations under the License. --> {% include JB/setup %} -## Customize your zeppelin homepage - Zeppelin allows you to use one of the notebooks you create as your zeppelin Homepage. - With that you can brand your zeppelin installation, - adjust the instruction to your users needs and even translate to other languages. +# Customize Apache Zeppelin homepage -
    -### How to set a notebook as your zeppelin homepage +
    -The process for creating your homepage is very simple as shown below: - - 1. Create a notebook using zeppelin - 2. Set the notebook id in the config file - 3. Restart zeppelin +Apache Zeppelin allows you to use one of the notebooks you create as your Zeppelin Homepage. +With that you can brand your Zeppelin installation, adjust the instruction to your users needs and even translate to other languages. -
    -#### Create a notebook using zeppelin - Create a new notebook using zeppelin, - you can use ```%md``` interpreter for markdown content or any other interpreter you like. +## How to set a notebook as your Zeppelin homepage - You can also use the display system to generate [text](../displaysystem/display.html), - [html](../displaysystem/display.html#html),[table](../displaysystem/table.html) or - [angular](../displaysystem/angular.html) +The process for creating your homepage is very simple as shown below: - Run (shift+Enter) the notebook and see the output. Optionally, change the notebook view to report to hide - the code sections. +1. Create a notebook using Zeppelin +2. Set the notebook id in the config file +3. Restart Zeppelin -
    -#### Set the notebook id in the config file - To set the notebook id in the config file you should copy it from the last word in the notebook url +### Create a notebook using Zeppelin +Create a new notebook using Zeppelin, +you can use ```%md``` interpreter for markdown content or any other interpreter you like. +You can also use the display system to generate [text](../displaysystem/basicdisplaysystem.html#text), [html](../displaysystem/basicdisplaysystem.html#html), [table](../displaysystem/basicdisplaysystem.html#table) or +Angular ([backend API](../displaysystem/back-end-angular.html), [frontend API](../displaysystem/front-end-angular.html)). - for example +Run (shift+Enter) the notebook and see the output. Optionally, change the notebook view to report to hide +the code sections. - +### Set the notebook id in the config file +To set the notebook id in the config file, you should copy it from the last word in the notebook url. +For example, - Set the notebook id to the ```ZEPPELIN_NOTEBOOK_HOMESCREEN``` environment variable - or ```zeppelin.notebook.homescreen``` property. + - You can also set the ```ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE``` environment variable - or ```zeppelin.notebook.homescreen.hide``` property to hide the new notebook from the notebook list. +Set the notebook id to the ```ZEPPELIN_NOTEBOOK_HOMESCREEN``` environment variable +or ```zeppelin.notebook.homescreen``` property. -
    -#### Restart zeppelin - Restart your zeppelin server +You can also set the ```ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE``` environment variable +or ```zeppelin.notebook.homescreen.hide``` property to hide the new notebook from the notebook list. - ``` - ./bin/zeppelin-deamon stop - ./bin/zeppelin-deamon start - ``` - ####That's it! Open your browser and navigate to zeppelin and see your customized homepage... +### Restart Zeppelin +Restart your Zeppelin server +``` +./bin/zeppelin-deamon stop +./bin/zeppelin-deamon start +``` +That's it! Open your browser and navigate to Apache Zeppelin and see your customized homepage.
    -### Show notebooks list in your custom homepage -If you want to display the list of notebooks on your custom zeppelin homepage all +## Show notebooks list in your custom homepage +If you want to display the list of notebooks on your custom Apache Zeppelin homepage all you need to do is use our %angular support. -
    - Add the following code to a paragraph in you home page and run it... walla! you have your notebooks list. - - ```javascript - println( - """%angular -
    -

    Notebooks

    - +Add the following code to a paragraph in you home page and run it... walla! you have your notebooks list. + +```javascript +println( +"""%angular +
    +

    Notebooks

    + - """) - ``` +
    +""") +``` - After running the notebook you will see output similar to this one: - +After running the notebook you will see output similar to this one: + - The main trick here relays in linking the ```
    ``` to the controller: +The main trick here relays in linking the ```
    ``` to the controller: - ```javascript -
    - ``` +```javascript +
    +``` - Once we have ```home``` as our controller variable in our ```
    ``` - we can use ```home.notes.list``` to get access to the notebook list. +Once we have ```home``` as our controller variable in our ```
    ``` +we can use ```home.notes.list``` to get access to the notebook list. diff --git a/docs/manual/publish.md b/docs/manual/publish.md index 1559fba2f8e..a6dea0d3c91 100644 --- a/docs/manual/publish.md +++ b/docs/manual/publish.md @@ -19,13 +19,14 @@ limitations under the License. --> {% include JB/setup %} -## How can you publish your paragraph ? -Zeppelin provides a feature for publishing your notebook paragraph results. Using this feature, you can show Zeppelin notebook paragraph results in your own website. -It's very straightforward. Just use `