diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 9ca740d139d..467c1d58733 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -105,7 +105,7 @@ zeppelin.interpreters - org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.tachyon.TachyonInterpreter + org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.tachyon.TachyonInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter Comma separated interpreter configurations. First interpreter become a default diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index d0581b14549..fac700defa1 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -43,6 +43,7 @@
  • Elasticsearch
  • Flink
  • Geode
    • +
  • HBase
  • Hive
  • Ignite
  • Lens
    • diff --git a/docs/interpreter/hbase.md b/docs/interpreter/hbase.md new file mode 100644 index 00000000000..30cec44b25a --- /dev/null +++ b/docs/interpreter/hbase.md @@ -0,0 +1,57 @@ +--- +layout: page +title: "HBase Shell Interpreter" +description: "" +group: manual +--- +{% include JB/setup %} + + +## HBase Shell Interpreter for Apache Zeppelin +[HBase Shell](http://hbase.apache.org/book.html#shell) is a JRuby IRB client for Apache HBase. +This interpreter provides all capabilities of Apache HBase shell within Apache Zeppelin. The +interpreter assumes that Apache HBase client software has been installed and its possible to +connect to the Apache HBase cluster from the machine on where Apache Zeppelin is installed. + +
    +## 1. Configuration + + + + + + + + + + + + + + + + + + + + + + +
    PropertyDefaultDescription
    hbase.home/usr/lib/hbaseInstallation directory of Hbase
    hbase.ruby.sourceslib/rubyPath to Ruby scripts relative to 'hbase.home'
    hbase.test.modefalseDisable checks for unit and manual tests
    + +## 2. Enabling the HBase Shell Interpreter + +In a notebook, to enable the **HBase Shell** interpreter, click the **Gear** icon and select +**HBase Shell**. + +## 3. Using the HBase Shell Interpreter + +In a paragraph, use `%hbase` to select the **HBase Shell** interpreter and then input all commands. + To get the list of available commands, use `help`. + +```bash +| %hbase +| help +``` + +For more information on all commands available, refer to [HBase Shell Documentation](http://hbase.apache.org/book.html#shell) \ No newline at end of file diff --git a/hbase/pom.xml b/hbase/pom.xml new file mode 100644 index 00000000000..f70ef88cf1e --- /dev/null +++ b/hbase/pom.xml @@ -0,0 +1,176 @@ + + + + 4.0.0 + + zeppelin + org.apache.zeppelin + 0.6.0-incubating-SNAPSHOT + + + org.apache.zeppelin + zeppelin-hbase + jar + 0.6.0-incubating-SNAPSHOT + Zeppelin: HBase interpreter + http://www.apache.org + + + 1.0.0 + 2.6.0 + 1.6.8 + 2.5.0 + + + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + provided + + + + org.apache.commons + commons-exec + 1.1 + + + + junit + junit + 4.11 + test + + + org.hamcrest + hamcrest-all + 1.3 + test + + + org.jruby + jruby-complete + ${jruby.version} + + + org.apache.hadoop + hadoop-yarn-common + ${hbase.hadoop.version} + + + org.apache.hadoop + hadoop-yarn-api + ${hbase.hadoop.version} + + + org.apache.hbase + hbase-client + ${hbase.hbase.version} + + + org.apache.hbase + hbase-annotations + ${hbase.hbase.version} + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + org.apache.hbase + hbase-server + ${hbase.hbase.version} + + + jline + jline + 2.12.1 + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + maven-enforcer-plugin + 1.3.1 + + + enforce + none + + + + + + maven-dependency-plugin + 2.8 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory}/../../interpreter/hbase + false + false + true + runtime + + + + copy-artifact + package + + copy + + + ${project.build.directory}/../../interpreter/hbase + false + false + true + runtime + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + + diff --git a/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java new file mode 100644 index 00000000000..dbcb33d4d99 --- /dev/null +++ b/hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java @@ -0,0 +1,158 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.hbase; + +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.jruby.embed.LocalContextScope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.jruby.embed.ScriptingContainer; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +/** + * Support for Hbase Shell. All the commands documented here + * http://hbase.apache.org/book.html#shell is supported. + * + * Requirements: + * Hbase Shell should be installed on the same machine. To be more specific, the following dir. + * should be available: https://github.com/apache/hbase/tree/master/hbase-shell/src/main/ruby + * Hbase Shell should be able to connect to the Hbase cluster from terminal. This makes sure + * that the client is configured properly. + * + * The interpreter takes 3 config parameters: + * hbase.home: Root dir. where hbase is installed. Default is /usr/lib/hbase/ + * hbase.ruby.sources: Dir where shell ruby code is installed. + * Path is relative to hbase.home. Default: lib/ruby + * hbase.irb.load: (Testing only) Default is true. + * Whether to load irb in the interpreter. + */ +public class HbaseInterpreter extends Interpreter { + private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class); + private ScriptingContainer scriptingContainer; + + private StringWriter writer; + + static { + Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(), + new InterpreterPropertyBuilder() + .add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase") + .add("hbase.ruby.sources", "lib/ruby", + "Path to Ruby scripts relative to 'hbase.home'") + .add("hbase.test.mode", "false", "Disable checks for unit and manual tests") + .build()); + } + + public HbaseInterpreter(Properties property) { + super(property); + } + + @Override + public void open() { + this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETON); + this.writer = new StringWriter(); + scriptingContainer.setOutput(this.writer); + + if (!Boolean.parseBoolean(getProperty("hbase.test.mode"))) { + String hbase_home = getProperty("hbase.home"); + String ruby_src = getProperty("hbase.ruby.sources"); + Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath(); + + logger.info("Home:" + hbase_home); + logger.info("Ruby Src:" + ruby_src); + + File f = abs_ruby_src.toFile(); + if (!f.exists() || !f.isDirectory()) { + throw new InterpreterException("hbase ruby sources is not available at '" + abs_ruby_src + + "'"); + } + + logger.info("Absolute Ruby Source:" + abs_ruby_src.toString()); + // hirb.rb:41 requires the following system property to be set. + Properties sysProps = System.getProperties(); + sysProps.setProperty("hbase.ruby.sources", abs_ruby_src.toString()); + + Path abs_hirb_path = Paths.get(hbase_home, "bin/hirb.rb"); + try { + FileInputStream fis = new FileInputStream(abs_hirb_path.toFile()); + this.scriptingContainer.runScriptlet(fis, "hirb.rb"); + fis.close(); + } catch (IOException e) { + throw new InterpreterException(e.getCause()); + } + } + } + + @Override + public void close() { + if (this.scriptingContainer != null) { + this.scriptingContainer.terminate(); + } + } + + @Override + public InterpreterResult interpret(String cmd, InterpreterContext interpreterContext) { + try { + logger.info(cmd); + this.writer.getBuffer().setLength(0); + this.scriptingContainer.runScriptlet(cmd); + this.writer.flush(); + logger.debug(writer.toString()); + return new InterpreterResult(InterpreterResult.Code.SUCCESS, writer.getBuffer().toString()); + } catch (Throwable t) { + logger.error("Can not run '" + cmd + "'", t); + return new InterpreterResult(InterpreterResult.Code.ERROR, t.getMessage()); + } + } + + @Override + public void cancel(InterpreterContext context) {} + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public Scheduler getScheduler() { + return SchedulerFactory.singleton().createOrGetFIFOScheduler( + HbaseInterpreter.class.getName() + this.hashCode()); + } + + @Override + public List completion(String buf, int cursor) { + return null; + } + +} diff --git a/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java new file mode 100644 index 00000000000..e2180702fce --- /dev/null +++ b/hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java @@ -0,0 +1,75 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.hbase; + +import org.apache.log4j.BasicConfigurator; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertEquals; + +/** + * Tests for HBase Interpreter + */ +public class HbaseInterpreterTest { + private static Logger logger = LoggerFactory.getLogger(HbaseInterpreterTest.class); + private static HbaseInterpreter hbaseInterpreter; + + @BeforeClass + public static void setUp() throws NullPointerException { + BasicConfigurator.configure(); + Properties properties = new Properties(); + properties.put("hbase.home", ""); + properties.put("hbase.ruby.sources", ""); + properties.put("hbase.test.mode", "true"); + + hbaseInterpreter = new HbaseInterpreter(properties); + hbaseInterpreter.open(); + } + + @Test + public void newObject() { + assertThat(hbaseInterpreter, notNullValue()); + } + + @Test + public void putsTest() { + InterpreterResult result = hbaseInterpreter.interpret("puts \"Hello World\"", null); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(result.type(), InterpreterResult.Type.TEXT); + assertEquals("Hello World\n", result.message()); + } + + public void putsLoadPath() { + InterpreterResult result = hbaseInterpreter.interpret("require 'two_power'; puts twoToThePowerOf(4)", null); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(result.type(), InterpreterResult.Type.TEXT); + assertEquals("16\n", result.message()); + } + + @Test + public void testException() { + InterpreterResult result = hbaseInterpreter.interpret("plot practical joke", null); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertEquals("(NameError) undefined local variable or method `joke' for main:Object", result.message()); + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index d73caf39a3b..736791f9f7b 100755 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ angular shell hive + hbase phoenix postgresql jdbc diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE index 36e983654bd..a12b9c44454 100644 --- a/zeppelin-distribution/src/bin_license/LICENSE +++ b/zeppelin-distribution/src/bin_license/LICENSE @@ -37,6 +37,7 @@ The following components are provided under Apache License. (Apache 2.0) Apache Cassandra (http://cassandra.apache.org/) (Apache 2.0) Apache CXF (http://cxf.apache.org/) (Apache 2.0) Apache Hive (http://hive.apache.org/) + (Apache 2.0) Apache HBase (http://hbase.apache.org/) (Apache 2.0) Apache Ignite (http://ignite.apache.org/) (Apache 2.0) Apache Kylin (http://kylin.apache.org/) (Apache 2.0) Apache Lens (http://lens.apache.org/) @@ -94,6 +95,7 @@ The following components are provided under Apache License. (Apache 2.0) Shiro Core (org.apache.shiro:shiro-core:1.2.3 - https://shiro.apache.org) (Apache 2.0) Shiro Web (org.apache.shiro:shiro-web:1.2.3 - https://shiro.apache.org) (Apache 2.0) SnakeYAML (org.yaml:snakeyaml:1.15 - http://www.snakeyaml.org) + (Apache 2.0) Protocol Buffers (com.google.protobuf:protobuf-java:2.4.1 - https://github.com/google/protobuf/releases) (Apache 2.0) Tachyon Shell (org.tachyonproject:tachyon-shell:0.8.2 - http://tachyon-project.org) (Apache 2.0) Tachyon Servers (org.tachyonproject:tachyon-servers:0.8.2 - http://tachyon-project.org) (Apache 2.0) Tachyon Minicluster (org.tachyonproject:tachyon-minicluster:0.8.2 - http://tachyon-project.org) @@ -154,6 +156,8 @@ The text of each license is also included at licenses/LICENSE-[project]-[version (BSD Style) dom4j v1.6.1 (http://www.dom4j.org) - https://github.com/dom4j/dom4j/blob/dom4j_1_6_1/LICENSE.txt (BSD Style) JSch v0.1.53 (http://www.jcraft.com) - http://www.jcraft.com/jsch/LICENSE.txt (BSD 3 Clause) highlightjs v8.4.0 (https://highlightjs.org/) - https://github.com/isagalaev/highlight.js/blob/8.4/LICENSE + (BSD 3 Clause) hamcrest v1.3 (http://hamcrest.org/JavaHamcrest/) - http://opensource.org/licenses/BSD-3-Clause + (BSD Style) JLine v2.12.1 (https://github.com/jline/jline2) - https://github.com/jline/jline2/blob/master/LICENSE.txt @@ -194,6 +198,7 @@ The following components are provided under the EPL License. (EPL 1.0) Aether (org.sonatype.aether - http://www.eclipse.org/aether/) (EPL 1.0) JDT Annotations For Enhanced Null Analysis (org.eclipse.jdt:org.eclipse.jdt.annotation:1.1.0 - https://repo.eclipse.org/content/repositories/eclipse-releases/org/eclipse/jdt/org.eclipse.jdt.annotation) + (EPL 1.0) JRuby (org.jruby.jruby-complete:v1.6.8 - http://www.jruby.org/) ======================================================================== diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 9e606ee2105..3088cfb7731 100755 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -454,7 +454,8 @@ public static enum ConfVars { + "org.apache.zeppelin.kylin.KylinInterpreter," + "org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter," + "org.apache.zeppelin.scalding.ScaldingInterpreter," - + "org.apache.zeppelin.jdbc.JDBCInterpreter"), + + "org.apache.zeppelin.jdbc.JDBCInterpreter," + + "org.apache.zeppelin.hbase.HbaseInterpreter"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000), ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),