From bdba5b25f47e359dc69c545deff9e4bd6e9f42a6 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Wed, 29 Apr 2015 10:10:45 +0530 Subject: [PATCH 01/29] Hbase Interpreter --- conf/zeppelin-site.xml.template | 2 +- hbase/pom.xml | 173 ++++++++++++++++ .../zeppelin/hbase/HbaseInterpreter.java | 130 ++++++++++++ hbase/src/main/resources/hbase/bin/hirb.rb | 189 ++++++++++++++++++ .../zeppelin/hbase/HbaseInterpreterTest.java | 75 +++++++ pom.xml | 1 + 6 files changed, 569 insertions(+), 1 deletion(-) create mode 100644 hbase/pom.xml create mode 100644 hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java create mode 100644 hbase/src/main/resources/hbase/bin/hirb.rb create mode 100644 hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 9f773d50add..e5e344cb895 100644 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -66,7 +66,7 @@ zeppelin.interpreters - org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter + org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter Comma separated interpreter configurations. First interpreter become a default diff --git a/hbase/pom.xml b/hbase/pom.xml new file mode 100644 index 00000000000..b4225623a7f --- /dev/null +++ b/hbase/pom.xml @@ -0,0 +1,173 @@ + + + + + zeppelin + org.apache.zeppelin + 0.5.0-SNAPSHOT + + 4.0.0 + Zeppelin: Hbase interpreter + zeppelin-hbase + + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + provided + + + + org.apache.commons + commons-exec + 1.1 + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + junit + junit + 4.11 + test + + + org.hamcrest + hamcrest-all + 1.3 + test + + + org.jruby + jruby-complete + 1.6.8 + + + org.apache.hadoop + hadoop-yarn-common + 2.6.0 + + + org.apache.hadoop + hadoop-yarn-api + 2.6.0 + + + org.apache.hbase + hbase-client + 1.0.0 + + + org.apache.hbase + hbase-annotations + 1.0.0 + + + com.google.protobuf + protobuf-java + 2.5.0 + + + org.apache.hbase + hbase-server + 1.0.0 + + + jline + jline + 2.12.1 + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + maven-enforcer-plugin + 1.3.1 + + + enforce + none + + + + + + maven-dependency-plugin + 2.8 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory}/../../interpreter/hbase + false + false + true + runtime + + + + copy-artifact + package + + copy + + + ${project.build.directory}/../../interpreter/hbase + false + false + true + runtime + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + + diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java new file mode 100644 index 00000000000..56770237e2f --- /dev/null +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.hbase; + +import org.apache.commons.exec.*; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.jruby.embed.LocalContextScope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.jruby.embed.ScriptingContainer; + +import java.io.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +/** + * Created by rajatv on 4/22/15. + */ +public class HbaseInterpreter extends Interpreter { + Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class); + int commandTimeOut = 600000; + ScriptingContainer scriptingContainer; + + StringWriter writer; + + static { + Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(), + new InterpreterPropertyBuilder() + .add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase") + .add("hbase.ruby.sources", "lib/ruby", + "Path to Ruby scripts relative to 'hbase.home'") + .add("hbase.irb.load", "true", "Load hirb. Optional for testing only") + .build()); + } + + public HbaseInterpreter(Properties property) { + super(property); + } + + @Override + public void open() { + String hbase_home = getProperty("hbase.home"); + String ruby_src = getProperty("hbase.ruby.sources"); + String abs_ruby_src = hbase_home + ruby_src; + + logger.info("Home:" + hbase_home); + logger.info("Ruby Src:" + ruby_src); + + Properties props = System.getProperties(); + props.setProperty("hbase.ruby.sources", abs_ruby_src); + this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETON); + List paths = new ArrayList<>(Arrays.asList(abs_ruby_src)); + this.writer = new StringWriter(); + scriptingContainer.setOutput(this.writer); + this.scriptingContainer.setLoadPaths(paths); + scriptingContainer.setCompatVersion(org.jruby.CompatVersion.RUBY1_9); + if (Boolean.parseBoolean(getProperty("hbase.irb.load"))) { + try { + InputStream in = getClass().getResourceAsStream("/hbase/bin/hirb.rb"); + scriptingContainer.runScriptlet(in, "/hbase/bin/hirb.rb"); + in.close(); + } catch (NullPointerException | IOException e) { + logger.error("Open failed:", e); + } + } + } + + @Override + public void close() {} + + @Override + public InterpreterResult interpret(String cmd, InterpreterContext interpreterContext) { + try { + logger.info(cmd); + this.writer.getBuffer().setLength(0); + this.scriptingContainer.runScriptlet(cmd); + this.writer.flush(); + logger.debug(writer.toString()); + return new InterpreterResult(InterpreterResult.Code.SUCCESS, writer.getBuffer().toString()); + } catch (Throwable t) { + logger.error("Can not run " + cmd, t); + return new InterpreterResult(InterpreterResult.Code.ERROR, t.getMessage()); + } + } + + @Override + public void cancel(InterpreterContext context) {} + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public Scheduler getScheduler() { + return SchedulerFactory.singleton().createOrGetFIFOScheduler( + HbaseInterpreter.class.getName() + this.hashCode()); + } + + @Override + public List completion(String buf, int cursor) { + return null; + } + +} diff --git a/hbase/src/main/resources/hbase/bin/hirb.rb b/hbase/src/main/resources/hbase/bin/hirb.rb new file mode 100644 index 00000000000..1de755e3f2f --- /dev/null +++ b/hbase/src/main/resources/hbase/bin/hirb.rb @@ -0,0 +1,189 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports +# and hbase commands and then loads jirb. Outputs a banner that tells user +# where to find help, shell version, and loads up a custom hirb. + +# TODO: Interrupt a table creation or a connection to a bad master. Currently +# has to time out. Below we've set down the retries for rpc and hbase but +# still can be annoying (And there seem to be times when we'll retry for +# ever regardless) +# TODO: Add support for listing and manipulating catalog tables, etc. +# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes + +# Run the java magic include and import basic HBase types that will help ease +# hbase hacking. +include Java + +# Some goodies for hirb. Should these be left up to the user's discretion? +require 'irb/completion' + +# +# FIXME: Switch args processing to getopt +# +# See if there are args for this shell. If any, read and then strip from ARGV +# so they don't go through to irb. Output shell 'usage' if user types '--help' +cmdline_help = <angular shell hive + hbase tajo zeppelin-web zeppelin-server From 03e1b7abf8426284f86f29cf567de1b1983c0145 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Sat, 2 May 2015 13:58:15 +0530 Subject: [PATCH 02/29] Fix test setup --- .../java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java index b3ea57c0e7e..5b41d21b08a 100644 --- a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java +++ b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java @@ -38,7 +38,7 @@ public class HbaseInterpreterTest { public static void setUp() throws NullPointerException { BasicConfigurator.configure(); Properties properties = new Properties(); - properties.put("hbase.home", HbaseInterpreterTest.class.getClassLoader().getResource("ruby/").toString()); + properties.put("hbase.home", ""); properties.put("hbase.ruby.sources", ""); properties.put("hbase.irb.load", "false"); logger.info("Resource: " + properties.getProperty("hbase.home")); From cd41ec29eb67a9b7851a718f21ddaf5864a69eb4 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Sat, 2 May 2015 14:11:18 +0530 Subject: [PATCH 03/29] Add a profile for Hbase-1.0 --- hbase/pom.xml | 14 +++++++------- pom.xml | 12 ++++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index b4225623a7f..0bf54712c3a 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -65,37 +65,37 @@ org.jruby jruby-complete - 1.6.8 + ${jruby.version} org.apache.hadoop hadoop-yarn-common - 2.6.0 + ${yarn.version} org.apache.hadoop hadoop-yarn-api - 2.6.0 + ${yarn.version} org.apache.hbase hbase-client - 1.0.0 + ${hbase.version} org.apache.hbase hbase-annotations - 1.0.0 + ${hbase.version} com.google.protobuf protobuf-java - 2.5.0 + ${protobuf.version} org.apache.hbase hbase-server - 1.0.0 + ${hbase.version} jline diff --git a/pom.xml b/pom.xml index 3c325d68889..d82e4091885 100644 --- a/pom.xml +++ b/pom.xml @@ -130,6 +130,7 @@ 1.8.8 1.0.5 4.0.17.Final + 1.6.8 15.0 2.2.1 @@ -1428,6 +1429,17 @@ + + hbase-1.0 + + 1.0.0 + 2.6.0 + 2.5.0 + 0.9.3 + 1.9.13 + + + mapr3 From 63a2d89a7e283ccb9ed3605879650c1556adc78d Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Fri, 22 May 2015 15:26:25 +0530 Subject: [PATCH 04/29] Add a new config for hbase version for interpreter --- hbase/pom.xml | 6 +++--- pom.xml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index 0bf54712c3a..29dc996531a 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -80,12 +80,12 @@ org.apache.hbase hbase-client - ${hbase.version} + ${hbase.interpreter.version} org.apache.hbase hbase-annotations - ${hbase.version} + ${hbase..interpreter.version} com.google.protobuf @@ -95,7 +95,7 @@ org.apache.hbase hbase-server - ${hbase.version} + ${hbase.interpreter.version} jline diff --git a/pom.xml b/pom.xml index d82e4091885..63b65b4ad58 100644 --- a/pom.xml +++ b/pom.xml @@ -131,6 +131,7 @@ 1.0.5 4.0.17.Final 1.6.8 + 1.0.0 15.0 2.2.1 From 032a2a4dbe99966b3488999b037bd3701c139663 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Fri, 22 May 2015 15:32:15 +0530 Subject: [PATCH 05/29] More fixes for version config vars --- hbase/pom.xml | 10 +++++----- pom.xml | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index 29dc996531a..361602cd969 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -70,22 +70,22 @@ org.apache.hadoop hadoop-yarn-common - ${yarn.version} + ${hbase.interpreter.hadoop} org.apache.hadoop hadoop-yarn-api - ${yarn.version} + ${hbase.interpreter.hadoop} org.apache.hbase hbase-client - ${hbase.interpreter.version} + ${hbase.interpreter.hbase} org.apache.hbase hbase-annotations - ${hbase..interpreter.version} + ${hbase.interpreter.hbase} com.google.protobuf @@ -95,7 +95,7 @@ org.apache.hbase hbase-server - ${hbase.interpreter.version} + ${hbase.interpreter.hbase} jline diff --git a/pom.xml b/pom.xml index 63b65b4ad58..fd545848518 100644 --- a/pom.xml +++ b/pom.xml @@ -131,7 +131,8 @@ 1.0.5 4.0.17.Final 1.6.8 - 1.0.0 + 1.0.0 + 2.6.0 15.0 2.2.1 From b2ff198b4601327db7a37bc0cb7a24ad69b4a4fe Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Fri, 22 May 2015 15:39:28 +0530 Subject: [PATCH 06/29] More fixes - sigh --- hbase/pom.xml | 25 ++++++++++++++++++------- pom.xml | 2 -- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index 361602cd969..06bd6a5ce03 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -18,14 +18,25 @@ + 4.0.0 + zeppelin org.apache.zeppelin 0.5.0-SNAPSHOT - 4.0.0 - Zeppelin: Hbase interpreter + + org.apache.zeppelin zeppelin-hbase + jar + 0.5.0-SNAPSHOT + Zeppelin: HBase interpreter + http://www.apache.org + + + 1.0.0 + 2.6.0 + @@ -70,22 +81,22 @@ org.apache.hadoop hadoop-yarn-common - ${hbase.interpreter.hadoop} + ${hbase.hadoop.version} org.apache.hadoop hadoop-yarn-api - ${hbase.interpreter.hadoop} + ${hbase.hadoop.version} org.apache.hbase hbase-client - ${hbase.interpreter.hbase} + ${hbase.hbase.version} org.apache.hbase hbase-annotations - ${hbase.interpreter.hbase} + ${hbase.hbase.version} com.google.protobuf @@ -95,7 +106,7 @@ org.apache.hbase hbase-server - ${hbase.interpreter.hbase} + ${hbase.hbase.version} jline diff --git a/pom.xml b/pom.xml index fd545848518..d82e4091885 100644 --- a/pom.xml +++ b/pom.xml @@ -131,8 +131,6 @@ 1.0.5 4.0.17.Final 1.6.8 - 1.0.0 - 2.6.0 15.0 2.2.1 From b9a407c3912fb76191296fc6c85548ed0254b523 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Fri, 22 May 2015 15:55:03 +0530 Subject: [PATCH 07/29] Specify relative path --- hbase/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/hbase/pom.xml b/hbase/pom.xml index 06bd6a5ce03..b560f6f55fb 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -24,6 +24,7 @@ zeppelin org.apache.zeppelin 0.5.0-SNAPSHOT + ../ org.apache.zeppelin From b7b06cd0c0eb105060ebf2b7f6d3f313aab7a8e5 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Fri, 22 May 2015 19:52:21 +0530 Subject: [PATCH 08/29] Fix version to add incubating string --- hbase/pom.xml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index b560f6f55fb..4337b20cc29 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -23,14 +23,13 @@ zeppelin org.apache.zeppelin - 0.5.0-SNAPSHOT - ../ + 0.5.0-incubating-SNAPSHOT org.apache.zeppelin zeppelin-hbase jar - 0.5.0-SNAPSHOT + 0.5.0-incubating-SNAPSHOT Zeppelin: HBase interpreter http://www.apache.org From 9a23a21727a5c8312ff5301e428470150a8a3d7a Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Fri, 22 May 2015 20:00:58 +0530 Subject: [PATCH 09/29] Add an explicit dependency for mapreduce-client-core --- hbase/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hbase/pom.xml b/hbase/pom.xml index 4337b20cc29..06a592fef34 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -88,6 +88,11 @@ hadoop-yarn-api ${hbase.hadoop.version} + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hbase.hadoop.version} + org.apache.hbase hbase-client From b8b52ba7d43b858029b06d6a06e5e1368608d6ac Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Wed, 2 Sep 2015 16:34:13 -0700 Subject: [PATCH 10/29] Missed a couple of resolved conflicts --- hbase/pom.xml | 4 +- pom.xml | 217 -------------------------------------------------- 2 files changed, 2 insertions(+), 219 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index d644f7af786..a041523cd41 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -23,13 +23,13 @@ zeppelin org.apache.zeppelin - 0.5.0-incubating-SNAPSHOT + 0.6.0-incubating-SNAPSHOT org.apache.zeppelin zeppelin-hbase jar - 0.5.0-incubating-SNAPSHOT + 0.6.0-incubating-SNAPSHOT Zeppelin: HBase interpreter http://www.apache.org diff --git a/pom.xml b/pom.xml index 7ee0f39fdd3..bea4a86dadb 100755 --- a/pom.xml +++ b/pom.xml @@ -641,222 +641,5 @@ -<<<<<<< HEAD - - - spark-1.1 - - - - - 1.1.1 - - - - - spark-1.2 - - - - 2.3.4-spark - 1.2.1 - 0.13.1a - 10.10.1.1 - 1.6.0rc3 - 0.5.0 - 4.2.6 - 3.1.1 - 4.0.23.Final - - - - - spark-1.3 - - - - 2.3.4-spark - 1.3.0 - 0.21.0 - 0.98.7 - hbase - org.spark-project.hive - 0.13.1a - 10.10.1.1 - 3.0.0.v201112011016 - 1.6.0rc3 - 0.5.0 - 2.4.0 - 2.0.8 - - 3.1.0 - 4.2.6 - 3.1.1 - 4.0.23.Final - 1.9.13 - 2.4.4 - 1.1.1.6 - 0.21.0 - - - - - hadoop-0.23 - - - - org.apache.avro - avro - - - - 0.23.10 - - - - - hadoop-2.2 - - 2.2.0 - 2.5.0 - - - - - hadoop-2.3 - - 2.3.0 - 2.5.0 - 0.9.0 - - - - - hadoop-2.4 - - 2.4.0 - 2.5.0 - 0.9.3 - - - - - hadoop-2.6 - - 2.6.0 - 2.5.0 - 0.9.3 - 1.9.13 - - - - - hbase-1.0 - - 1.0.0 - 2.6.0 - 2.5.0 - 0.9.3 - 1.9.13 - - - - - mapr3 - - false - - - 1.0.3-mapr-3.0.3 - 2.3.0-mapr-4.0.0-FCS - 0.94.17-mapr-1405 - 3.4.5-mapr-1406 - - - - - mapr4 - - false - - - 2.3.0-mapr-4.0.0-FCS - 2.3.0-mapr-4.0.0-FCS - 0.94.17-mapr-1405-4.0.0-FCS - 3.4.5-mapr-1406 - - - - org.apache.curator - curator-recipes - 2.4.0 - - - org.apache.zookeeper - zookeeper - - - - - org.apache.zookeeper - zookeeper - 3.4.5-mapr-1406 - - - - - - - hadoop-provided - - false - - - - org.apache.hadoop - hadoop-client - provided - - - org.apache.hadoop - hadoop-yarn-api - provided - - - org.apache.hadoop - hadoop-yarn-common - provided - - - org.apache.hadoop - hadoop-yarn-server-web-proxy - provided - - - org.apache.hadoop - hadoop-yarn-client - provided - - - org.apache.avro - avro - provided - - - org.apache.avro - avro-ipc - provided - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - provided - - - -======= ->>>>>>> master From feacd892296be440dc0e085ed911b1df70564578 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Wed, 2 Sep 2015 16:40:35 -0700 Subject: [PATCH 11/29] Fix tab in pom.xml --- hbase/pom.xml | 323 +++++++++++++++++++++++++------------------------- 1 file changed, 161 insertions(+), 162 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index a041523cd41..5d83c28659a 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -16,176 +16,175 @@ ~ limitations under the License. --> - 4.0.0 - - - zeppelin - org.apache.zeppelin - 0.6.0-incubating-SNAPSHOT - - + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + zeppelin org.apache.zeppelin - zeppelin-hbase - jar 0.6.0-incubating-SNAPSHOT - Zeppelin: HBase interpreter - http://www.apache.org + + + org.apache.zeppelin + zeppelin-hbase + jar + 0.6.0-incubating-SNAPSHOT + Zeppelin: HBase interpreter + http://www.apache.org - - 1.0.0 - 2.6.0 - 1.6.8 - 2.4.1 - + + 1.0.0 + 2.6.0 + 1.6.8 + 2.4.1 + - - - ${project.groupId} - zeppelin-interpreter - ${project.version} - provided - + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + provided + - - org.apache.commons - commons-exec - 1.1 - + + org.apache.commons + commons-exec + 1.1 + - - org.slf4j - slf4j-api - + + org.slf4j + slf4j-api + - - org.slf4j - slf4j-log4j12 - - - junit - junit - 4.11 - test - - - org.hamcrest - hamcrest-all - 1.3 - test - - - org.jruby - jruby-complete - ${jruby.version} - - - org.apache.hadoop - hadoop-yarn-common - ${hbase.hadoop.version} - - - org.apache.hadoop - hadoop-yarn-api - ${hbase.hadoop.version} - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hbase.hadoop.version} - - - org.apache.hbase - hbase-client - ${hbase.hbase.version} - - - org.apache.hbase - hbase-annotations - ${hbase.hbase.version} - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - org.apache.hbase - hbase-server - ${hbase.hbase.version} - - - jline - jline - 2.12.1 - - + + org.slf4j + slf4j-log4j12 + + + junit + junit + 4.11 + test + + + org.hamcrest + hamcrest-all + 1.3 + test + + + org.jruby + jruby-complete + ${jruby.version} + + + org.apache.hadoop + hadoop-yarn-common + ${hbase.hadoop.version} + + + org.apache.hadoop + hadoop-yarn-api + ${hbase.hadoop.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hbase.hadoop.version} + + + org.apache.hbase + hbase-client + ${hbase.hbase.version} + + + org.apache.hbase + hbase-annotations + ${hbase.hbase.version} + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + org.apache.hbase + hbase-server + ${hbase.hbase.version} + + + jline + jline + 2.12.1 + + - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - - true - - + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + - - maven-enforcer-plugin - 1.3.1 - - - enforce - none - - - + + maven-enforcer-plugin + 1.3.1 + + + enforce + none + + + - - maven-dependency-plugin - 2.8 - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/hbase - false - false - true - runtime - - - - copy-artifact - package - - copy - - - ${project.build.directory}/../../interpreter/hbase - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - - - - + + maven-dependency-plugin + 2.8 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory}/../../interpreter/hbase + false + false + true + runtime + + + + copy-artifact + package + + copy + + + ${project.build.directory}/../../interpreter/hbase + false + false + true + runtime + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + From c160787ed5fb3fde423105f4b9862e52958a7dc3 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Wed, 2 Sep 2015 16:54:13 -0700 Subject: [PATCH 12/29] Explicitly add private keyword. Improve javadoc --- .../apache/zeppelin/hbase/HbaseInterpreter.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 56770237e2f..b07995fc8a3 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -14,7 +14,6 @@ package org.apache.zeppelin.hbase; -import org.apache.commons.exec.*; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; @@ -27,21 +26,22 @@ import org.jruby.embed.ScriptingContainer; -import java.io.*; +import java.io.InputStream; +import java.io.IOException; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Properties; /** - * Created by rajatv on 4/22/15. + * HBase Shell Interpreter. (https://wiki.apache.org/hadoop/Hbase/Shell) */ public class HbaseInterpreter extends Interpreter { - Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class); - int commandTimeOut = 600000; - ScriptingContainer scriptingContainer; + private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class); + private ScriptingContainer scriptingContainer; - StringWriter writer; + private StringWriter writer; static { Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(), From 19f137ebc34241656ed75f8820df091c452a4b85 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Wed, 2 Sep 2015 17:02:44 -0700 Subject: [PATCH 13/29] Terminate jruby --- .../java/org/apache/zeppelin/hbase/HbaseInterpreter.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index b07995fc8a3..759c10c50b1 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -86,7 +86,11 @@ public void open() { } @Override - public void close() {} + public void close() { + if (this.scriptingContainer != null) { + this.scriptingContainer.terminate(); + } + } @Override public InterpreterResult interpret(String cmd, InterpreterContext interpreterContext) { From b598b70548fc9811d115d749239b04a5b1f84f62 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Wed, 2 Sep 2015 17:04:20 -0700 Subject: [PATCH 14/29] Fix javadoc in unit tests --- .../java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java index 5b41d21b08a..fca5afeb82e 100644 --- a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java +++ b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java @@ -28,7 +28,7 @@ import static org.junit.Assert.assertEquals; /** - * Created by rajatv on 4/24/15. + * Tests for HBase Interpreter */ public class HbaseInterpreterTest { private static Logger logger = LoggerFactory.getLogger(HbaseInterpreterTest.class); From 255d0d904e8a63a37e23b4ba45b712554225b52d Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Thu, 3 Sep 2015 11:14:51 -0700 Subject: [PATCH 15/29] Add more info in javadoc --- .../apache/zeppelin/hbase/HbaseInterpreter.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 759c10c50b1..d2de445baad 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -35,7 +35,21 @@ import java.util.Properties; /** - * HBase Shell Interpreter. (https://wiki.apache.org/hadoop/Hbase/Shell) + * Support for Hbase Shell. All the commands documented here + * https://wiki.apache.org/hadoop/Hbase/Shell is supported. + * + * Requirements: + * Hbase Shell should be installed on the same machine. To be more specific, the following dir. + * should be available: https://github.com/apache/hbase/tree/master/hbase-shell/src/main/ruby + * Hbase Shell should be able to connect to the Hbase cluster from terminal. This makes sure + * that the client is configured properly. + * + * The interpreter takes 3 config parameters: + * hbase.home: Root dir. where hbase is installed. Default is /usr/lib/hbase/ + * hbase.ruby.sources: Dir where shell ruby code is installed. + * Path is relative to hbase.home. Default: lib/ruby + * hbase.irb.load: (Testing only) Default is true. + * Whether to load irb in the interpreter. */ public class HbaseInterpreter extends Interpreter { private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class); From ca649310c05847c63dbf8850020491f5f2f58e85 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Mon, 7 Sep 2015 13:07:36 -0700 Subject: [PATCH 16/29] Review comments --- .../apache/zeppelin/hbase/HbaseInterpreter.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index d2de445baad..398e79ec8f0 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -14,10 +14,7 @@ package org.apache.zeppelin.hbase; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; -import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.jruby.embed.LocalContextScope; @@ -26,6 +23,7 @@ import org.jruby.embed.ScriptingContainer; +import java.io.File; import java.io.InputStream; import java.io.IOException; import java.io.StringWriter; @@ -63,7 +61,8 @@ public class HbaseInterpreter extends Interpreter { .add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase") .add("hbase.ruby.sources", "lib/ruby", "Path to Ruby scripts relative to 'hbase.home'") - .add("hbase.irb.load", "true", "Load hirb. Optional for testing only") + .add("hbase.irb.load", "true", "Load hirb. hirb loads hbase shell. " + + "For unit tests it has to be turned off.") .build()); } @@ -77,6 +76,11 @@ public void open() { String ruby_src = getProperty("hbase.ruby.sources"); String abs_ruby_src = hbase_home + ruby_src; + File f = new File(abs_ruby_src); + if (!f.exists() || !f.isDirectory()) { + throw new InterpreterException("hbase ruby sources is not correctly defined."); + } + logger.info("Home:" + hbase_home); logger.info("Ruby Src:" + ruby_src); From 7e9dffb984d515d038e5da32ed0898522c265a7c Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Sat, 9 Jan 2016 22:35:35 +0530 Subject: [PATCH 17/29] Minor review comments --- .../main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 398e79ec8f0..820821efd0f 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -92,6 +92,8 @@ public void open() { scriptingContainer.setOutput(this.writer); this.scriptingContainer.setLoadPaths(paths); scriptingContainer.setCompatVersion(org.jruby.CompatVersion.RUBY1_9); + + // The following block is used for manual tests only. if (Boolean.parseBoolean(getProperty("hbase.irb.load"))) { try { InputStream in = getClass().getResourceAsStream("/hbase/bin/hirb.rb"); @@ -120,7 +122,7 @@ public InterpreterResult interpret(String cmd, InterpreterContext interpreterCon logger.debug(writer.toString()); return new InterpreterResult(InterpreterResult.Code.SUCCESS, writer.getBuffer().toString()); } catch (Throwable t) { - logger.error("Can not run " + cmd, t); + logger.error("Can not run '" + cmd + "'", t); return new InterpreterResult(InterpreterResult.Code.ERROR, t.getMessage()); } } From e21539b20b720dc0380ab0f49eae1618f9216edf Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Mon, 11 Jan 2016 05:53:18 +0530 Subject: [PATCH 18/29] Remove hirb. Add a property for testing which disables all checks --- .../zeppelin/hbase/HbaseInterpreter.java | 17 +- hbase/src/main/resources/hbase/bin/hirb.rb | 189 ------------------ .../zeppelin/hbase/HbaseInterpreterTest.java | 4 +- 3 files changed, 5 insertions(+), 205 deletions(-) delete mode 100644 hbase/src/main/resources/hbase/bin/hirb.rb diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 820821efd0f..979938e8e7a 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -61,8 +61,7 @@ public class HbaseInterpreter extends Interpreter { .add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase") .add("hbase.ruby.sources", "lib/ruby", "Path to Ruby scripts relative to 'hbase.home'") - .add("hbase.irb.load", "true", "Load hirb. hirb loads hbase shell. " + - "For unit tests it has to be turned off.") + .add("hbase.test.mode", "false", "Disable checks for unit and manual tests") .build()); } @@ -77,7 +76,8 @@ public void open() { String abs_ruby_src = hbase_home + ruby_src; File f = new File(abs_ruby_src); - if (!f.exists() || !f.isDirectory()) { + if (!Boolean.parseBoolean(getProperty("hbase.test.mode")) && + (!f.exists() || !f.isDirectory())) { throw new InterpreterException("hbase ruby sources is not correctly defined."); } @@ -92,17 +92,6 @@ public void open() { scriptingContainer.setOutput(this.writer); this.scriptingContainer.setLoadPaths(paths); scriptingContainer.setCompatVersion(org.jruby.CompatVersion.RUBY1_9); - - // The following block is used for manual tests only. - if (Boolean.parseBoolean(getProperty("hbase.irb.load"))) { - try { - InputStream in = getClass().getResourceAsStream("/hbase/bin/hirb.rb"); - scriptingContainer.runScriptlet(in, "/hbase/bin/hirb.rb"); - in.close(); - } catch (NullPointerException | IOException e) { - logger.error("Open failed:", e); - } - } } @Override diff --git a/hbase/src/main/resources/hbase/bin/hirb.rb b/hbase/src/main/resources/hbase/bin/hirb.rb deleted file mode 100644 index 1de755e3f2f..00000000000 --- a/hbase/src/main/resources/hbase/bin/hirb.rb +++ /dev/null @@ -1,189 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports -# and hbase commands and then loads jirb. Outputs a banner that tells user -# where to find help, shell version, and loads up a custom hirb. - -# TODO: Interrupt a table creation or a connection to a bad master. Currently -# has to time out. Below we've set down the retries for rpc and hbase but -# still can be annoying (And there seem to be times when we'll retry for -# ever regardless) -# TODO: Add support for listing and manipulating catalog tables, etc. -# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes - -# Run the java magic include and import basic HBase types that will help ease -# hbase hacking. -include Java - -# Some goodies for hirb. Should these be left up to the user's discretion? -require 'irb/completion' - -# -# FIXME: Switch args processing to getopt -# -# See if there are args for this shell. If any, read and then strip from ARGV -# so they don't go through to irb. Output shell 'usage' if user types '--help' -cmdline_help = < Date: Mon, 11 Jan 2016 08:40:11 +0530 Subject: [PATCH 19/29] Use Paths and Path instead of Strings to manipulate file system paths --- .../zeppelin/hbase/HbaseInterpreter.java | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 979938e8e7a..76acc1cfff7 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -27,6 +27,8 @@ import java.io.InputStream; import java.io.IOException; import java.io.StringWriter; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -71,27 +73,28 @@ public HbaseInterpreter(Properties property) { @Override public void open() { - String hbase_home = getProperty("hbase.home"); - String ruby_src = getProperty("hbase.ruby.sources"); - String abs_ruby_src = hbase_home + ruby_src; - - File f = new File(abs_ruby_src); - if (!Boolean.parseBoolean(getProperty("hbase.test.mode")) && - (!f.exists() || !f.isDirectory())) { - throw new InterpreterException("hbase ruby sources is not correctly defined."); - } - - logger.info("Home:" + hbase_home); - logger.info("Ruby Src:" + ruby_src); - - Properties props = System.getProperties(); - props.setProperty("hbase.ruby.sources", abs_ruby_src); this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETON); - List paths = new ArrayList<>(Arrays.asList(abs_ruby_src)); this.writer = new StringWriter(); scriptingContainer.setOutput(this.writer); - this.scriptingContainer.setLoadPaths(paths); scriptingContainer.setCompatVersion(org.jruby.CompatVersion.RUBY1_9); + + if (!Boolean.parseBoolean(getProperty("hbase.test.mode"))) { + String hbase_home = getProperty("hbase.home"); + String ruby_src = getProperty("hbase.ruby.sources"); + Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath(); + + logger.info("Home:" + hbase_home); + logger.info("Ruby Src:" + ruby_src); + + File f = abs_ruby_src.toFile(); + if (!f.exists() || !f.isDirectory()) { + throw new InterpreterException("hbase ruby sources is not available at '" + abs_ruby_src + + "'"); + } + + List paths = new ArrayList<>(Arrays.asList(abs_ruby_src.toAbsolutePath().toString())); + this.scriptingContainer.setLoadPaths(paths); + } } @Override From 7078a35fc711d6a1c7169e1e238992f49dec60c5 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Wed, 20 Jan 2016 18:50:46 +0530 Subject: [PATCH 20/29] Add a doc for HBase shell interpreter --- docs/interpreter/hbase.md | 57 +++++++++++++++++++ .../zeppelin/hbase/HbaseInterpreter.java | 2 +- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 docs/interpreter/hbase.md diff --git a/docs/interpreter/hbase.md b/docs/interpreter/hbase.md new file mode 100644 index 00000000000..30cec44b25a --- /dev/null +++ b/docs/interpreter/hbase.md @@ -0,0 +1,57 @@ +--- +layout: page +title: "HBase Shell Interpreter" +description: "" +group: manual +--- +{% include JB/setup %} + + +## HBase Shell Interpreter for Apache Zeppelin +[HBase Shell](http://hbase.apache.org/book.html#shell) is a JRuby IRB client for Apache HBase. +This interpreter provides all capabilities of Apache HBase shell within Apache Zeppelin. The +interpreter assumes that Apache HBase client software has been installed and its possible to +connect to the Apache HBase cluster from the machine on where Apache Zeppelin is installed. + +
+## 1. Configuration + + + + + + + + + + + + + + + + + + + + + + +
PropertyDefaultDescription
hbase.home/usr/lib/hbaseInstallation directory of Hbase
hbase.ruby.sourceslib/rubyPath to Ruby scripts relative to 'hbase.home'
hbase.test.modefalseDisable checks for unit and manual tests
+ +## 2. Enabling the HBase Shell Interpreter + +In a notebook, to enable the **HBase Shell** interpreter, click the **Gear** icon and select +**HBase Shell**. + +## 3. Using the HBase Shell Interpreter + +In a paragraph, use `%hbase` to select the **HBase Shell** interpreter and then input all commands. + To get the list of available commands, use `help`. + +```bash +| %hbase +| help +``` + +For more information on all commands available, refer to [HBase Shell Documentation](http://hbase.apache.org/book.html#shell) \ No newline at end of file diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 76acc1cfff7..3d87c68aacb 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -36,7 +36,7 @@ /** * Support for Hbase Shell. All the commands documented here - * https://wiki.apache.org/hadoop/Hbase/Shell is supported. + * http://hbase.apache.org/book.html#shell is supported. * * Requirements: * Hbase Shell should be installed on the same machine. To be more specific, the following dir. From 320bfd5117f1ca7d268850550e80aadba5dc7e93 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Thu, 21 Jan 2016 09:51:42 +0530 Subject: [PATCH 21/29] Add information about licenses --- zeppelin-distribution/src/bin_license/LICENSE | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE index a5324d0c486..8f85ec0dede 100644 --- a/zeppelin-distribution/src/bin_license/LICENSE +++ b/zeppelin-distribution/src/bin_license/LICENSE @@ -37,6 +37,7 @@ The following components are provided under Apache License. (Apache 2.0) Apache Cassandra (http://cassandra.apache.org/) (Apache 2.0) Apache CXF (http://cxf.apache.org/) (Apache 2.0) Apache Hive (http://hive.apache.org/) + (Apache 2.0) Apache HBase (http://hbase.apache.org/) (Apache 2.0) Apache Ignite (http://ignite.apache.org/) (Apache 2.0) Apache Kylin (http://kylin.apache.org/) (Apache 2.0) Apache Lens (http://lens.apache.org/) @@ -94,6 +95,7 @@ The following components are provided under Apache License. (Apache 2.0) Shiro Core (org.apache.shiro:shiro-core:1.2.3 - https://shiro.apache.org) (Apache 2.0) Shiro Web (org.apache.shiro:shiro-web:1.2.3 - https://shiro.apache.org) (Apache 2.0) SnakeYAML (org.yaml:snakeyaml:1.15 - http://www.snakeyaml.org) + (Apache 2.0) Protocol Buffers (com.google.protobuf:protobuf-java:2.4.1 - https://github.com/google/protobuf/releases) @@ -150,6 +152,8 @@ The text of each license is also included at licenses/LICENSE-[project]-[version (BSD Style) dom4j v1.6.1 (http://www.dom4j.org) - https://github.com/dom4j/dom4j/blob/dom4j_1_6_1/LICENSE.txt (BSD Style) JSch v0.1.53 (http://www.jcraft.com) - http://www.jcraft.com/jsch/LICENSE.txt (BSD 3 Clause) highlightjs v8.4.0 (https://highlightjs.org/) - https://github.com/isagalaev/highlight.js/blob/8.4/LICENSE + (BSD 3 Clause) hamcrest v1.3 (http://hamcrest.org/JavaHamcrest/) - http://opensource.org/licenses/BSD-3-Clause + (BSD Style) JLine v2.12.1 (https://github.com/jline/jline2) - https://github.com/jline/jline2/blob/master/LICENSE.txt @@ -190,6 +194,7 @@ The following components are provided under the EPL License. (EPL 1.0) Aether (org.sonatype.aether - http://www.eclipse.org/aether/) (EPL 1.0) JDT Annotations For Enhanced Null Analysis (org.eclipse.jdt:org.eclipse.jdt.annotation:1.1.0 - https://repo.eclipse.org/content/repositories/eclipse-releases/org/eclipse/jdt/org.eclipse.jdt.annotation) + (EPL 1.0) JRuby (org.jruby.jruby-complete:v1.6.8 - http://www.jruby.org/) ======================================================================== From b4ad18ddf1b72aa02eb5593dc266479aeaeb66b1 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Sun, 31 Jan 2016 13:45:45 +0530 Subject: [PATCH 22/29] Load hirb --- .../org/apache/zeppelin/hbase/HbaseInterpreter.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 3d87c68aacb..c71371e6ed5 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -24,6 +24,8 @@ import org.jruby.embed.ScriptingContainer; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.InputStream; import java.io.IOException; import java.io.StringWriter; @@ -94,6 +96,15 @@ public void open() { List paths = new ArrayList<>(Arrays.asList(abs_ruby_src.toAbsolutePath().toString())); this.scriptingContainer.setLoadPaths(paths); + + Path abs_hirb_path = Paths.get(hbase_home, ruby_src, "irb/hirb.rb"); + try { + FileInputStream fis = new FileInputStream(abs_hirb_path.toFile()); + this.scriptingContainer.runScriptlet(fis, "hirb.rb"); + fis.close(); + } catch (IOException e) { + throw new InterpreterException(e.getCause()); + } } } From 39dddb827f3c3c9c67da3f3b6d47e360e1f23e0e Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Sun, 31 Jan 2016 13:53:50 +0530 Subject: [PATCH 23/29] Address minor review comments --- docs/_includes/themes/zeppelin/_navigation.html | 1 + .../java/org/apache/zeppelin/conf/ZeppelinConfiguration.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index d0581b14549..fac700defa1 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -43,6 +43,7 @@
  • Elasticsearch
  • Flink
  • Geode
    • +
  • HBase
  • Hive
  • Ignite
  • Lens
    • diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 9e606ee2105..13963d52e02 100755 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -454,7 +454,8 @@ public static enum ConfVars { + "org.apache.zeppelin.kylin.KylinInterpreter," + "org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter," + "org.apache.zeppelin.scalding.ScaldingInterpreter," - + "org.apache.zeppelin.jdbc.JDBCInterpreter"), + + "org.apache.zeppelin.jdbc.JDBCInterpreter," + + "org.apache.zeppelin.jdbc.HbaseInterpreter"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000), ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10), From 57ebad8d4cab08f456a75d528281ac1fb9bc8b00 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Sun, 31 Jan 2016 14:25:27 +0530 Subject: [PATCH 24/29] Fix hirb path --- .../main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index c71371e6ed5..cd5f8459b40 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -97,7 +97,7 @@ public void open() { List paths = new ArrayList<>(Arrays.asList(abs_ruby_src.toAbsolutePath().toString())); this.scriptingContainer.setLoadPaths(paths); - Path abs_hirb_path = Paths.get(hbase_home, ruby_src, "irb/hirb.rb"); + Path abs_hirb_path = Paths.get(hbase_home, "bin/hirb.rb"); try { FileInputStream fis = new FileInputStream(abs_hirb_path.toFile()); this.scriptingContainer.runScriptlet(fis, "hirb.rb"); From 69755ad7b3295faa8edd9546bb1d2aa9229091cc Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Sun, 31 Jan 2016 14:44:26 +0530 Subject: [PATCH 25/29] Set hbase.ruby.sources in system properties --- .../main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index cd5f8459b40..76f3a3ec402 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -97,6 +97,10 @@ public void open() { List paths = new ArrayList<>(Arrays.asList(abs_ruby_src.toAbsolutePath().toString())); this.scriptingContainer.setLoadPaths(paths); + // hirb.rb:41 requires the following system property to be set. + Properties sysProps = System.getProperties(); + sysProps.setProperty("hbase.ruby.sources", abs_ruby_src.toString()); + Path abs_hirb_path = Paths.get(hbase_home, "bin/hirb.rb"); try { FileInputStream fis = new FileInputStream(abs_hirb_path.toFile()); From 9924784ed6349acd3628ded07d4350608405fd89 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Sun, 31 Jan 2016 17:36:37 +0530 Subject: [PATCH 26/29] Log absolute path of ruby source --- .../main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 76f3a3ec402..6320c9a059c 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -97,10 +97,11 @@ public void open() { List paths = new ArrayList<>(Arrays.asList(abs_ruby_src.toAbsolutePath().toString())); this.scriptingContainer.setLoadPaths(paths); + logger.info("Absolute Ruby Source:" + abs_ruby_src.toString()); // hirb.rb:41 requires the following system property to be set. Properties sysProps = System.getProperties(); sysProps.setProperty("hbase.ruby.sources", abs_ruby_src.toString()); - + Path abs_hirb_path = Paths.get(hbase_home, "bin/hirb.rb"); try { FileInputStream fis = new FileInputStream(abs_hirb_path.toFile()); From 3efc892bb2d2f3c1f77122d4d5f9f66bf7973d6c Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Sun, 31 Jan 2016 18:26:34 +0530 Subject: [PATCH 27/29] Do not use ruby 1.9 and do not bother setting paths --- .../main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java index 6320c9a059c..dbcb33d4d99 100644 --- a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -78,7 +78,6 @@ public void open() { this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETON); this.writer = new StringWriter(); scriptingContainer.setOutput(this.writer); - scriptingContainer.setCompatVersion(org.jruby.CompatVersion.RUBY1_9); if (!Boolean.parseBoolean(getProperty("hbase.test.mode"))) { String hbase_home = getProperty("hbase.home"); @@ -94,9 +93,6 @@ public void open() { + "'"); } - List paths = new ArrayList<>(Arrays.asList(abs_ruby_src.toAbsolutePath().toString())); - this.scriptingContainer.setLoadPaths(paths); - logger.info("Absolute Ruby Source:" + abs_ruby_src.toString()); // hirb.rb:41 requires the following system property to be set. Properties sysProps = System.getProperties(); From c17db3b4bfc7f6a7be5e33fa814f56bac2b93b85 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Mon, 1 Feb 2016 12:32:40 +0530 Subject: [PATCH 28/29] Fix typo in ZeppelinConfiguration --- .../java/org/apache/zeppelin/conf/ZeppelinConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 13963d52e02..3088cfb7731 100755 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -455,7 +455,7 @@ public static enum ConfVars { + "org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter," + "org.apache.zeppelin.scalding.ScaldingInterpreter," + "org.apache.zeppelin.jdbc.JDBCInterpreter," - + "org.apache.zeppelin.jdbc.HbaseInterpreter"), + + "org.apache.zeppelin.hbase.HbaseInterpreter"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000), ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10), From caf87f3df4161d55609deb47a6b0d8f2af938a20 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh Date: Mon, 1 Feb 2016 16:26:35 +0530 Subject: [PATCH 29/29] Fix Protobuf version number --- hbase/pom.xml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/hbase/pom.xml b/hbase/pom.xml index 5d83c28659a..f70ef88cf1e 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -36,7 +36,7 @@ 1.0.0 2.6.0 1.6.8 - 2.4.1 + 2.5.0 @@ -53,15 +53,6 @@ 1.1     - - org.slf4j - slf4j-api - - - - org.slf4j - slf4j-log4j12 - junit junit @@ -89,11 +80,6 @@ hadoop-yarn-api ${hbase.hadoop.version} - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hbase.hadoop.version} - org.apache.hbase hbase-client