From 38e2556a76d0e258218e913e3249100f072a5827 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 17 Jun 2016 20:32:59 -0700 Subject: [PATCH 01/22] Initial implementation of install-interpreter.sh --- bin/install-interpreter.sh | 43 +++++ .../zeppelin/dep/DependencyResolver.java | 22 +++ .../install/InstallInterpreter.java | 161 ++++++++++++++++++ 3 files changed, 226 insertions(+) create mode 100755 bin/install-interpreter.sh create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java diff --git a/bin/install-interpreter.sh b/bin/install-interpreter.sh new file mode 100755 index 00000000000..7b072d4bd7f --- /dev/null +++ b/bin/install-interpreter.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Run Zeppelin +# + +bin=$(dirname "${BASH_SOURCE-$0}") +bin=$(cd "${bin}">/dev/null; pwd) + +. "${bin}/common.sh" + + +ZEPPELIN_INSTALL_INTERPRETER_MAIN=org.apache.zeppelin.interpreter.install.InstallInterpreter + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes" +fi +addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" +fi +addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" + +addJarInDir "${ZEPPELIN_HOME}/lib" + +CLASSPATH+=":${ZEPPELIN_CLASSPATH}" +$ZEPPELIN_RUNNER $JAVA_OPTS -cp $CLASSPATH $ZEPPELIN_INSTALL_INTERPRETER_MAIN ${@} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index 60ef1f9e6e0..94cae4ec876 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -113,6 +113,28 @@ public List load(String artifact, Collection excludes, String dest return libs; } + public List load(String artifact, File destPath) throws IOException, RepositoryException { + return load(artifact, new LinkedList(), destPath); + } + + public List load(String artifact, Collection excludes, File destPath) + throws RepositoryException, IOException { + List libs = new LinkedList(); + + if (StringUtils.isNotBlank(artifact)) { + libs = load(artifact, excludes); + + for (File srcFile : libs) { + File destFile = new File(destPath, srcFile.getName()); + if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) { + FileUtils.copyFile(srcFile, destFile); + logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath); + } + } + } + return libs; + } + private List loadFromMvn(String artifact, Collection excludes) throws RepositoryException { Collection allExclusions = new LinkedList(); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java new file mode 100644 index 00000000000..2035ff6712a --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.install; + +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Logger; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.util.Util; +import org.sonatype.aether.RepositoryException; +import java.io.File; +import java.io.IOException; +import java.util.Locale; +import java.util.Map; + +/** + * Commandline utility to install interpreter from maven repository + */ +public class InstallInterpreter { + static ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + + static String [] availableInterpreters = { + "md", "zeppelin-markdown", "Markdown interpreter", + "sh", "zeppelin-shell", "Allows shell command", + "jdbc", "zeppelin-jdbc", + "Generic JDBC interpreter for hive, phoenix, postsgresql, mysql, etc", + }; + + public static void list() { + for (int i = 0; i < availableInterpreters.length; i++) { + System.out.println(availableInterpreters[i] + "\t\t" + availableInterpreters[i + 2]); + i += 2; + } + } + + public static void installAll() { + for (int i = 0; i < availableInterpreters.length; i++) { + install(availableInterpreters[i], getArtifactName(availableInterpreters[i + 1])); + i += 2; + } + } + + public static void install(String name) { + // find artifact name + for (int i = 0; i < availableInterpreters.length; i++) { + if (name.equals(availableInterpreters[i])) { + install(name, getArtifactName(availableInterpreters[i + 1])); + return; + } + i += 2; + } + + System.err.println("Can't find interpreter '" + name + "'"); + System.exit(1); + } + + private static String getArtifactName(String name) { + return "org.apache.zeppelin:" + name + ":" + Util.getVersion(); + } + + public static void install(String name, String artifact) { + DependencyResolver depResolver = new DependencyResolver( + conf.getInterpreterLocalRepoPath()); + + File interpreterBaseDir = new File(conf.getInterpreterDir()); + File installDir = new File(interpreterBaseDir, name); + if (installDir.exists()) { + System.err.println("Directory " + installDir.getAbsolutePath() + " already exists. Skipping"); + return; + } + + System.out.println("Install " + name + "(" + artifact + ") to " + + installDir.getAbsolutePath() + " ... "); + + try { + depResolver.load(artifact, installDir); + System.out.println("Interpreter " + name + " installed under " + + installDir.getAbsolutePath()); + } catch (RepositoryException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public static void usage() { + System.out.println("Options"); + System.out.println(" -l, --list List available interpreters"); + System.out.println(" -a, --all Install all available interpreters"); + System.out.println(" -n, --name [NAME] Install an interpreter"); + System.out.println(" -t, --artifact [ARTIFACT] (Optional with -n) custom artifact name." + + "e.g. customGroup:customArtifact:customVersion"); + } + + public static void main(String [] args) { + if (args.length == 0) { + usage(); + return; + } + + String name = null; + String artifact = null; + + for (int i = 0; i < args.length; i++) { + String arg = args[i].toLowerCase(Locale.US); + switch (arg) { + case "--list": + case "-l": + list(); + System.exit(0); + break; + case "--all": + case "-a": + installAll(); + System.exit(0); + break; + case "--name": + case "-n": + name = args[++i]; + break; + case "--artifact": + case "-t": + artifact = args[++i]; + break; + case "--version": + case "-v": + Util.getVersion(); + break; + case "--help": + case "-h": + usage(); + System.exit(0); + break; + default: + System.out.println("Unknown option " + arg); + } + } + + if (name != null) { + if (artifact != null) { + install(name, artifact); + } else { + install(name); + } + } + } +} From 47f5706f0e173a6a5b11fa164476cde6b812f9a7 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 10:43:34 -0700 Subject: [PATCH 02/22] Install multiple interpreters at once --- .../install/InstallInterpreter.java | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java index 2035ff6712a..99096a537c5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java @@ -54,6 +54,12 @@ public static void installAll() { } } + public static void install(String [] names) { + for (String name : names) { + install(name); + } + } + public static void install(String name) { // find artifact name for (int i = 0; i < availableInterpreters.length; i++) { @@ -72,6 +78,17 @@ private static String getArtifactName(String name) { return "org.apache.zeppelin:" + name + ":" + Util.getVersion(); } + public static void install(String [] names, String [] artifacts) { + if (names.length != artifacts.length) { + System.err.println("Length of given names and artifacts are different"); + System.exit(1); + } + + for (int i = 0; i < names.length; i++) { + install(names[i], artifacts[i]); + } + } + public static void install(String name, String artifact) { DependencyResolver depResolver = new DependencyResolver( conf.getInterpreterLocalRepoPath()); @@ -101,8 +118,10 @@ public static void usage() { System.out.println("Options"); System.out.println(" -l, --list List available interpreters"); System.out.println(" -a, --all Install all available interpreters"); - System.out.println(" -n, --name [NAME] Install an interpreter"); - System.out.println(" -t, --artifact [ARTIFACT] (Optional with -n) custom artifact name." + + System.out.println(" -n, --name [NAMES] Install interpreters (comma separated list)" + + "e.g. spark,md,shell"); + System.out.println(" -t, --artifact [ARTIFACT] (Optional with -n) custom artifact names. " + + "(comma separated list correspond to --name) " + "e.g. customGroup:customArtifact:customVersion"); } @@ -112,8 +131,8 @@ public static void main(String [] args) { return; } - String name = null; - String artifact = null; + String names = null; + String artifacts = null; for (int i = 0; i < args.length; i++) { String arg = args[i].toLowerCase(Locale.US); @@ -130,11 +149,11 @@ public static void main(String [] args) { break; case "--name": case "-n": - name = args[++i]; + names = args[++i]; break; case "--artifact": case "-t": - artifact = args[++i]; + artifacts = args[++i]; break; case "--version": case "-v": @@ -150,11 +169,11 @@ public static void main(String [] args) { } } - if (name != null) { - if (artifact != null) { - install(name, artifact); + if (names != null) { + if (artifacts != null) { + install(names.split(","), artifacts.split(",")); } else { - install(name); + install(names.split(",")); } } } From 78a7c52f8931518fa188e2a3ae40ad981f639fe6 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 11:37:59 -0700 Subject: [PATCH 03/22] Refactor and add test --- .../zeppelin/conf/ZeppelinConfiguration.java | 4 + .../install/InstallInterpreter.java | 144 +++++++++++++----- .../install/InstallInterpreterTest.java | 86 +++++++++++ 3 files changed, 193 insertions(+), 41 deletions(-) create mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 45fbba4d57d..0a7b8c0121e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -346,6 +346,10 @@ public String getS3EncryptionMaterialsProviderClass() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP); } + public String getInterpreterListPath() { + return getRelativeDir(String.format("%s/interpreter-list", getConfDir())); + } + public String getInterpreterDir() { return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java index 99096a537c5..298c070abe9 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java @@ -16,6 +16,7 @@ */ package org.apache.zeppelin.interpreter.install; +import org.apache.commons.io.FileUtils; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Logger; import org.apache.zeppelin.conf.ZeppelinConfiguration; @@ -24,64 +25,121 @@ import org.sonatype.aether.RepositoryException; import java.io.File; import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Commandline utility to install interpreter from maven repository */ public class InstallInterpreter { - static ZeppelinConfiguration conf = ZeppelinConfiguration.create(); - - static String [] availableInterpreters = { - "md", "zeppelin-markdown", "Markdown interpreter", - "sh", "zeppelin-shell", "Allows shell command", - "jdbc", "zeppelin-jdbc", - "Generic JDBC interpreter for hive, phoenix, postsgresql, mysql, etc", - }; - - public static void list() { - for (int i = 0; i < availableInterpreters.length; i++) { - System.out.println(availableInterpreters[i] + "\t\t" + availableInterpreters[i + 2]); - i += 2; + private final File interpreterListFile; + private final File interpreterBaseDir; + private final List availableInterpreters; + private final String localRepoDir; + + /** + * + * @param interpreterListFile + * @param interpreterBaseDir interpreter directory for installing binaries + * @throws IOException + */ + public InstallInterpreter(File interpreterListFile, File interpreterBaseDir, String localRepoDir) + throws IOException { + this.interpreterListFile = interpreterListFile; + this.interpreterBaseDir = interpreterBaseDir; + this.localRepoDir = localRepoDir; + availableInterpreters = new LinkedList(); + readAvailableInterpreters(); + } + + + /** + * Information for available informations + */ + private static class AvailableInterpreterInfo { + public final String name; + public final String artifact; + public final String description; + + public AvailableInterpreterInfo(String name, String artifact, String description) { + this.name = name; + this.artifact = artifact; + this.description = description; + } + } + + private void readAvailableInterpreters() throws IOException { + if (!interpreterListFile.isFile()) { + System.err.print("Can't find interpreter list " + interpreterListFile.getAbsolutePath()); + return; + } + String text = FileUtils.readFileToString(interpreterListFile); + String[] lines = text.split("\n"); + + Pattern pattern = Pattern.compile("(\\S+)\\s+(\\S+)\\s+(.*)"); + + int lineNo = 0; + for (String line : lines) { + lineNo++; + if (line == null || line.length() == 0 || line.startsWith("#")) { + continue; + } + + Matcher match = pattern.matcher(line); + if (match.groupCount() != 3) { + System.err.println("Error on line " + lineNo + ", " + line); + continue; + } + + match.find(); + + String name = match.group(1); + String artifact = match.group(2); + String description = match.group(3); + + availableInterpreters.add(new AvailableInterpreterInfo(name, artifact, description)); } } - public static void installAll() { - for (int i = 0; i < availableInterpreters.length; i++) { - install(availableInterpreters[i], getArtifactName(availableInterpreters[i + 1])); - i += 2; + public List list() { + for (AvailableInterpreterInfo info : availableInterpreters) { + System.out.println(info.name + "\t\t" + info.description); } + + return availableInterpreters; } - public static void install(String [] names) { + public void installAll() { + for (AvailableInterpreterInfo info : availableInterpreters) { + install(info.name, info.artifact); + } + } + + public void install(String [] names) { for (String name : names) { install(name); } } - public static void install(String name) { + public void install(String name) { // find artifact name - for (int i = 0; i < availableInterpreters.length; i++) { - if (name.equals(availableInterpreters[i])) { - install(name, getArtifactName(availableInterpreters[i + 1])); + for (AvailableInterpreterInfo info : availableInterpreters) { + if (name.equals(info.name)) { + install(name, info.artifact); return; } - i += 2; } - System.err.println("Can't find interpreter '" + name + "'"); - System.exit(1); - } - - private static String getArtifactName(String name) { - return "org.apache.zeppelin:" + name + ":" + Util.getVersion(); + throw new RuntimeException("Can't find interpreter '" + name + "'"); } - public static void install(String [] names, String [] artifacts) { + public void install(String [] names, String [] artifacts) { if (names.length != artifacts.length) { - System.err.println("Length of given names and artifacts are different"); - System.exit(1); + throw new RuntimeException("Length of given names and artifacts are different"); } for (int i = 0; i < names.length; i++) { @@ -89,11 +147,9 @@ public static void install(String [] names, String [] artifacts) { } } - public static void install(String name, String artifact) { - DependencyResolver depResolver = new DependencyResolver( - conf.getInterpreterLocalRepoPath()); + public void install(String name, String artifact) { + DependencyResolver depResolver = new DependencyResolver(localRepoDir); - File interpreterBaseDir = new File(conf.getInterpreterDir()); File installDir = new File(interpreterBaseDir, name); if (installDir.exists()) { System.err.println("Directory " + installDir.getAbsolutePath() + " already exists. Skipping"); @@ -125,12 +181,18 @@ public static void usage() { "e.g. customGroup:customArtifact:customVersion"); } - public static void main(String [] args) { + public static void main(String [] args) throws IOException { if (args.length == 0) { usage(); return; } + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + InstallInterpreter installer = new InstallInterpreter( + new File(conf.getInterpreterListPath()), + new File(conf.getInterpreterDir()), + conf.getInterpreterLocalRepoPath()); + String names = null; String artifacts = null; @@ -139,12 +201,12 @@ public static void main(String [] args) { switch (arg) { case "--list": case "-l": - list(); + installer.list(); System.exit(0); break; case "--all": case "-a": - installAll(); + installer.installAll(); System.exit(0); break; case "--name": @@ -171,9 +233,9 @@ public static void main(String [] args) { if (names != null) { if (artifacts != null) { - install(names.split(","), artifacts.split(",")); + installer.install(names.split(","), artifacts.split(",")); } else { - install(names.split(",")); + installer.install(names.split(",")); } } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java new file mode 100644 index 00000000000..e934f1a0a1c --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java @@ -0,0 +1,86 @@ +package org.apache.zeppelin.interpreter.install; + +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class InstallInterpreterTest { + private File tmpDir; + private InstallInterpreter installer; + private File interpreterBaseDir; + + @Before + public void setUp() throws IOException { + tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + new File(tmpDir, "conf").mkdirs(); + interpreterBaseDir = new File(tmpDir, "interpreter"); + File localRepoDir = new File(tmpDir, "local-repo"); + interpreterBaseDir.mkdir(); + localRepoDir.mkdir(); + + File interpreterListFile = new File(tmpDir, "conf/interpreter-list"); + + + // create interpreter list file + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); + + String interpreterList = ""; + interpreterList += "intp1 org.apache.commons:commons-csv:1.1 test interpreter 1\n"; + interpreterList += "intp2 org.apache.commons:commons-math3:3.6.1 test interpreter 2\n"; + + FileUtils.writeStringToFile(new File(tmpDir, "conf/interpreter-list"), interpreterList); + + installer = new InstallInterpreter(interpreterListFile, interpreterBaseDir, localRepoDir + .getAbsolutePath()); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(tmpDir); + } + + + @Test + public void testList() { + assertEquals(2, installer.list().size()); + } + + @Test + public void install() { + assertEquals(0, interpreterBaseDir.listFiles().length); + + installer.install("intp1"); + assertTrue(new File(interpreterBaseDir, "intp1").isDirectory()); + } + + @Test + public void installAll() { + installer.installAll(); + assertTrue(new File(interpreterBaseDir, "intp1").isDirectory()); + assertTrue(new File(interpreterBaseDir, "intp2").isDirectory()); + } +} From 2c81a3f98b83d855ed012d5e395918864db1bda3 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 12:20:18 -0700 Subject: [PATCH 04/22] update --- bin/install-interpreter.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/install-interpreter.sh b/bin/install-interpreter.sh index 7b072d4bd7f..06be75cbf44 100755 --- a/bin/install-interpreter.sh +++ b/bin/install-interpreter.sh @@ -26,9 +26,11 @@ bin=$(cd "${bin}">/dev/null; pwd) ZEPPELIN_INSTALL_INTERPRETER_MAIN=org.apache.zeppelin.interpreter.install.InstallInterpreter +ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/install-interpreter.log" +JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}" -if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then - ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes" +if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" fi addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" From ec7d152549524b44e0019d900331d5e5ed710224 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 12:20:37 -0700 Subject: [PATCH 05/22] add tip --- .../interpreter/install/InstallInterpreter.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java index 298c070abe9..115056847ae 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java @@ -74,7 +74,7 @@ public AvailableInterpreterInfo(String name, String artifact, String description private void readAvailableInterpreters() throws IOException { if (!interpreterListFile.isFile()) { - System.err.print("Can't find interpreter list " + interpreterListFile.getAbsolutePath()); + System.err.println("Can't find interpreter list " + interpreterListFile.getAbsolutePath()); return; } String text = FileUtils.readFileToString(interpreterListFile); @@ -234,9 +234,23 @@ public static void main(String [] args) throws IOException { if (names != null) { if (artifacts != null) { installer.install(names.split(","), artifacts.split(",")); + configurationTip(); + interpreterSettingTip(); } else { installer.install(names.split(",")); + interpreterSettingTip(); } } } + + private static void configurationTip() { + System.out.println("Add interpreter class name to '" + + ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName() + "' property " + + "in your conf/zeppelin-site.xml file"); + } + + private static void interpreterSettingTip() { + System.out.println("Create interpreter setting in 'Interpreter' menu on GUI." + + "And then you can bind interpreter to your notebook to use"); + } } From 1b558fd145ae24eaae615487b560de1e17baa216 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 14:04:57 -0700 Subject: [PATCH 06/22] update some text --- .../interpreter/install/InstallInterpreter.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java index 115056847ae..f70ee27d6e5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java @@ -107,7 +107,7 @@ private void readAvailableInterpreters() throws IOException { public List list() { for (AvailableInterpreterInfo info : availableInterpreters) { - System.out.println(info.name + "\t\t" + info.description); + System.out.println(info.name + "\t\t\t" + info.description); } return availableInterpreters; @@ -162,7 +162,7 @@ public void install(String name, String artifact) { try { depResolver.load(artifact, installDir); System.out.println("Interpreter " + name + " installed under " + - installDir.getAbsolutePath()); + installDir.getAbsolutePath() + "."); } catch (RepositoryException e) { e.printStackTrace(); } catch (IOException e) { @@ -176,7 +176,7 @@ public static void usage() { System.out.println(" -a, --all Install all available interpreters"); System.out.println(" -n, --name [NAMES] Install interpreters (comma separated list)" + "e.g. spark,md,shell"); - System.out.println(" -t, --artifact [ARTIFACT] (Optional with -n) custom artifact names. " + + System.out.println(" -t, --artifact [ARTIFACTS] (Optional with -n) custom artifact names. " + "(comma separated list correspond to --name) " + "e.g. customGroup:customArtifact:customVersion"); } @@ -234,15 +234,21 @@ public static void main(String [] args) throws IOException { if (names != null) { if (artifacts != null) { installer.install(names.split(","), artifacts.split(",")); + startTip(); configurationTip(); interpreterSettingTip(); } else { installer.install(names.split(",")); + startTip(); interpreterSettingTip(); } } } + private static void startTip() { + System.out.println(""); + } + private static void configurationTip() { System.out.println("Add interpreter class name to '" + ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName() + "' property " @@ -251,6 +257,6 @@ private static void configurationTip() { private static void interpreterSettingTip() { System.out.println("Create interpreter setting in 'Interpreter' menu on GUI." - + "And then you can bind interpreter to your notebook to use"); + + " And then you can bind interpreter on your notebook"); } } From 49f0568ac25b5dc13e326a2edf28142deb6639c7 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 14:05:56 -0700 Subject: [PATCH 07/22] Add conf/interpreter-list --- conf/interpreter-list | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 conf/interpreter-list diff --git a/conf/interpreter-list b/conf/interpreter-list new file mode 100644 index 00000000000..d6eb45e4607 --- /dev/null +++ b/conf/interpreter-list @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# [name] [maven artifact] [description] + +alluxio org.apache.zeppelin.zeppelin-alluxio:0.6.0 Alluxio interpreter +angular org.apache.zeppelin.zeppelin-angular:0.6.0 HTML and AngularJS view rendering +cassandra org.apache.zeppelin.zeppelin-cassandra:0.6.0 Cassandra interpreter +elasticsearch org.apache.zeppelin.zeppelin-cassandra:0.6.0 Elasticsearch interpreter +file org.apache.zeppelin:zeppelin-file:0.6.0 HDFS file interpreter +flink org.apache.zeppelin:zeppelin-flink:0.6.0 Flink interpreter +hbase org.apache.zeppelin:zeppelin-hbase:0.6.0 Hbase interpreter +ignite org.apache.zeppelin:zeppelin-ignite:0.6.0 Ignite interpreter +jdbc org.apache.zeppelin:zeppelin-jdbc:0.6.0 Jdbc interpreter +kylin org.apache.zeppelin:zeppelin-kylin:0.6.0 Kylin interpreter +lens org.apache.zeppelin:zeppelin-lens:0.6.0 Lens interpreter +livy org.apache.zeppelin:zeppelin-livy:0.6.0 Livy interpreter +md org.apache.zeppelin:zeppelin-markdown:0.6.0 Markdown support +postgresql org.apache.zeppelin:zeppelin-postgresql:0.6.0 Postgresql interpreter +python org.apache.zeppelin:zeppelin-python:0.6.0 Python interpreter +shell org.apache.zeppelin:zeppelin-shell:0.6.0 Shell command +test1 org.apache.commons:commons-csv:1.1 test +test2 org.apache.commons:commons-csv:1.1 test + + + From 7b1b36accdfd790262905ff90085dd67f3fafd4f Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 14:11:00 -0700 Subject: [PATCH 08/22] update usage --- .../apache/zeppelin/interpreter/install/InstallInterpreter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java index f70ee27d6e5..800d163f289 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java @@ -175,7 +175,7 @@ public static void usage() { System.out.println(" -l, --list List available interpreters"); System.out.println(" -a, --all Install all available interpreters"); System.out.println(" -n, --name [NAMES] Install interpreters (comma separated list)" + - "e.g. spark,md,shell"); + "e.g. md,shell,jdbc,python,angular"); System.out.println(" -t, --artifact [ARTIFACTS] (Optional with -n) custom artifact names. " + "(comma separated list correspond to --name) " + "e.g. customGroup:customArtifact:customVersion"); From 6ee06b836a2bc2e1ba99f0c3c0dc9e3bddc171c9 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 20:00:41 -0700 Subject: [PATCH 09/22] Make DependencyResolver in zeppelin-interpreter module not aware of ZEPPELIN_HOME --- .../zeppelin/dep/DependencyResolver.java | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index 94cae4ec876..2fb058c6abf 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -63,11 +63,6 @@ public List load(String artifact) return load(artifact, new LinkedList()); } - public List load(String artifact, String destPath) - throws RepositoryException, IOException { - return load(artifact, new LinkedList(), destPath); - } - public synchronized List load(String artifact, Collection excludes) throws RepositoryException, IOException { if (StringUtils.isBlank(artifact)) { @@ -85,33 +80,6 @@ public synchronized List load(String artifact, Collection excludes return libs; } } - - public List load(String artifact, Collection excludes, String destPath) - throws RepositoryException, IOException { - List libs = new LinkedList(); - - if (StringUtils.isNotBlank(artifact)) { - libs = load(artifact, excludes); - - // find home dir - String home = System.getenv("ZEPPELIN_HOME"); - if (home == null) { - home = System.getProperty("zeppelin.home"); - } - if (home == null) { - home = ".."; - } - - for (File srcFile : libs) { - File destFile = new File(home + "/" + destPath, srcFile.getName()); - if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) { - FileUtils.copyFile(srcFile, destFile); - logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath); - } - } - } - return libs; - } public List load(String artifact, File destPath) throws IOException, RepositoryException { return load(artifact, new LinkedList(), destPath); From d547551e54e460d05814bded7db3bcb53092f649 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 20:03:47 -0700 Subject: [PATCH 10/22] Remove test entries --- conf/interpreter-list | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conf/interpreter-list b/conf/interpreter-list index d6eb45e4607..469cb8f3912 100644 --- a/conf/interpreter-list +++ b/conf/interpreter-list @@ -33,8 +33,3 @@ md org.apache.zeppelin:zeppelin-markdown:0.6.0 Markdown postgresql org.apache.zeppelin:zeppelin-postgresql:0.6.0 Postgresql interpreter python org.apache.zeppelin:zeppelin-python:0.6.0 Python interpreter shell org.apache.zeppelin:zeppelin-shell:0.6.0 Shell command -test1 org.apache.commons:commons-csv:1.1 test -test2 org.apache.commons:commons-csv:1.1 test - - - From aebca17ccc65419db72b666472b9aa6ecdbd4ba8 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 21:06:20 -0700 Subject: [PATCH 11/22] Add docs --- .../themes/zeppelin/_navigation.html | 2 +- docs/install/install.md | 55 +++++++++++++++++++ .../interpreter/InterpreterFactory.java | 6 +- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index 40b9fb6024b..e93ca52b765 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -21,7 +21,7 @@
  • What is Apache Zeppelin ?
  • Getting Started
  • -
  • Quick Start
  • +
  • Install
  • Configuration
  • Explore Zeppelin UI
  • Tutorial
  • diff --git a/docs/install/install.md b/docs/install/install.md index 93d9febe507..1a057b0c1e0 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -62,6 +62,8 @@ Although it can be unstable somehow since it is on development status, you can e If you want to install Apache Zeppelin with a stable binary package, please visit [Apache Zeppelin download Page](http://zeppelin.apache.org/download.html). After unpacking, jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section. +If you have downloaded `netinst` binary, [install additional interpreters](http://localhost:4000/install/install.html#install-interpreters) or simply run `./bin/install-interpreter.sh --all` before you jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section. + ### Building from Source If you want to build from the source, the software below needs to be installed on your system. @@ -389,3 +391,56 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z Size in characters of the maximum text message to be received by websocket. + +## Install interpreters + +You can install additional [interpreters](../manual/interpreters.html) using `bin/install-interpreter.sh` command. + +### Community managed interpreters + +Informations of community managed interpreters are listed in `conf/interpreter-list` file. `bin/install-interpreter.sh` command will read this file to install interpreters. + +##### Install all community managed interpreters + +``` +./bin/install-interpreter.sh --all +``` + +##### Install specific interpreters + +``` +./bin/install-interpreter.sh --name md,shell,jdbc,python +``` + +You can get full list of community managed interpreters by running + +``` +./bin/install-interpreter.sh --list +``` + +Once you have installed interpreter, restart Zeppelin. And then you'll need [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [binding it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). + + +### 3rd party interpreters + +`./bin/install-interpreter.sh` command can install 3rd party interpreters available in maven repository. + +##### Install 3rd party interpreters + +``` +./bin/install-interpreter.sh --name interpreter1 --repository groupId1:artifact1:version1 +``` + +The command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory. + +Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](install.html#zeppelin-configuration). +And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [binding it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). + + +##### Install multiple 3rd party interpreters at once + +`--name` and `--repository` argument receives comma separated list + +``` +./bin/install-interpreter.sh --name interpreter1,interpreter2 --repository groupId1:artifact1:version1,groupId2:artifact2:version2 +``` diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index bad18c02347..aeb78187777 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -350,15 +350,17 @@ private void loadInterpreterDependencies(InterpreterSetting intSetting) List deps = intSetting.getDependencies(); if (deps != null) { for (Dependency d: deps) { + File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); + if (d.getExclusions() != null) { depResolver.load( d.getGroupArtifactVersion(), d.getExclusions(), - conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id()); + new File(destDir, intSetting.id())); } else { depResolver.load( d.getGroupArtifactVersion(), - conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id()); + new File(destDir, intSetting.id())); } } } From 107729614d2049a66b6637f595aeb5de290e8a86 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 22:56:46 -0700 Subject: [PATCH 12/22] update test --- .../zeppelin/dep/DependencyResolverTest.java | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java index a8664ef67fe..af2c7ff6963 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java @@ -33,27 +33,20 @@ public class DependencyResolverTest { private static DependencyResolver resolver; private static String testPath; - private static String testCopyPath; - private static String home; - + private static File testCopyPath; + private static File tmpDir; + @BeforeClass public static void setUp() throws Exception { - testPath = "test-repo"; - testCopyPath = "test-copy-repo"; + tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + testPath = tmpDir.getAbsolutePath() + "/test-repo"; + testCopyPath = new File(tmpDir, "test-copy-repo"); resolver = new DependencyResolver(testPath); - home = System.getenv("ZEPPELIN_HOME"); - if (home == null) { - home = System.getProperty("zeppelin.home"); - } - if (home == null) { - home = ".."; - } } @AfterClass public static void tearDown() throws Exception { - FileUtils.deleteDirectory(new File(home + "/" + testPath)); - FileUtils.deleteDirectory(new File(home + "/" + testCopyPath)); + FileUtils.deleteDirectory(tmpDir); } @Rule @@ -78,19 +71,19 @@ public void testDelRepo() { public void testLoad() throws Exception { // basic load resolver.load("com.databricks:spark-csv_2.10:1.3.0", testCopyPath); - assertEquals(new File(home + "/" + testCopyPath).list().length, 4); - FileUtils.cleanDirectory(new File(home + "/" + testCopyPath)); + assertEquals(testCopyPath.list().length, 4); + FileUtils.cleanDirectory(testCopyPath); // load with exclusions parameter resolver.load("com.databricks:spark-csv_2.10:1.3.0", Collections.singletonList("org.scala-lang:scala-library"), testCopyPath); - assertEquals(new File(home + "/" + testCopyPath).list().length, 3); - FileUtils.cleanDirectory(new File(home + "/" + testCopyPath)); + assertEquals(testCopyPath.list().length, 3); + FileUtils.cleanDirectory(testCopyPath); // load from added repository resolver.addRepo("sonatype", "https://oss.sonatype.org/content/repositories/agimatec-releases/", false); resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath); - assertEquals(new File(home + "/" + testCopyPath).list().length, 8); + assertEquals(testCopyPath.list().length, 8); // load invalid artifact resolver.delRepo("sonatype"); From 9079580ff502984d2facf27ac829f77b39842b80 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 23:11:51 -0700 Subject: [PATCH 13/22] fix artifact name --- conf/interpreter-list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/interpreter-list b/conf/interpreter-list index 469cb8f3912..94940716793 100644 --- a/conf/interpreter-list +++ b/conf/interpreter-list @@ -20,7 +20,7 @@ alluxio org.apache.zeppelin.zeppelin-alluxio:0.6.0 Alluxio interpreter angular org.apache.zeppelin.zeppelin-angular:0.6.0 HTML and AngularJS view rendering cassandra org.apache.zeppelin.zeppelin-cassandra:0.6.0 Cassandra interpreter -elasticsearch org.apache.zeppelin.zeppelin-cassandra:0.6.0 Elasticsearch interpreter +elasticsearch org.apache.zeppelin.zeppelin-elasticsearch:0.6.0 Elasticsearch interpreter file org.apache.zeppelin:zeppelin-file:0.6.0 HDFS file interpreter flink org.apache.zeppelin:zeppelin-flink:0.6.0 Flink interpreter hbase org.apache.zeppelin:zeppelin-hbase:0.6.0 Hbase interpreter From 4c1f0290784a28c22cbbff38385d83733274f56a Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 23:12:35 -0700 Subject: [PATCH 14/22] Proxy support --- .../dep/AbstractDependencyResolver.java | 12 +++++ .../zeppelin/dep/DependencyResolver.java | 1 + .../install/InstallInterpreter.java | 51 ++++++++++++++++--- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java index b22941ef6b2..1e0844efaac 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.dep; +import java.net.URL; import java.util.Collection; import java.util.Iterator; import java.util.LinkedList; @@ -25,6 +26,7 @@ import org.sonatype.aether.RepositorySystem; import org.sonatype.aether.RepositorySystemSession; import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.Proxy; import org.sonatype.aether.repository.RemoteRepository; import org.sonatype.aether.repository.RepositoryPolicy; import org.sonatype.aether.resolution.ArtifactResult; @@ -44,6 +46,16 @@ public AbstractDependencyResolver(String localRepoPath) { repos.add(Booter.newLocalRepository()); } + public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) { + Authentication auth = new Authentication(proxyUser, proxyPassword); + Proxy proxy = new Proxy(proxyUrl.getProtocol(), proxyUrl.getHost(), proxyUrl.getPort(), auth); + synchronized (repos) { + for (RemoteRepository repo : repos) { + repo.setProxy(proxy); + } + } + } + public List getRepos() { return this.repos; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index 2fb058c6abf..214175a2640 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; +import java.net.URL; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java index 800d163f289..da67d9f1249 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java @@ -25,6 +25,7 @@ import org.sonatype.aether.RepositoryException; import java.io.File; import java.io.IOException; +import java.net.URL; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -40,6 +41,9 @@ public class InstallInterpreter { private final File interpreterBaseDir; private final List availableInterpreters; private final String localRepoDir; + private URL proxyUrl; + private String proxyUser; + private String proxyPassword; /** * @@ -149,6 +153,9 @@ public void install(String [] names, String [] artifacts) { public void install(String name, String artifact) { DependencyResolver depResolver = new DependencyResolver(localRepoDir); + if (proxyUrl != null) { + depResolver.setProxy(proxyUrl, proxyUser, proxyPassword); + } File installDir = new File(interpreterBaseDir, name); if (installDir.exists()) { @@ -170,15 +177,26 @@ public void install(String name, String artifact) { } } + public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) { + this.proxyUrl = proxyUrl; + this.proxyUser = proxyUser; + this.proxyPassword = proxyPassword; + } + public static void usage() { System.out.println("Options"); - System.out.println(" -l, --list List available interpreters"); - System.out.println(" -a, --all Install all available interpreters"); - System.out.println(" -n, --name [NAMES] Install interpreters (comma separated list)" + + System.out.println(" -l, --list List available interpreters"); + System.out.println(" -a, --all Install all available interpreters"); + System.out.println(" -n, --name [NAMES] Install interpreters (comma separated " + + "list)" + "e.g. md,shell,jdbc,python,angular"); - System.out.println(" -t, --artifact [ARTIFACTS] (Optional with -n) custom artifact names. " + + System.out.println(" -t, --artifact [ARTIFACTS] (Optional with -n) custom artifact names" + + ". " + "(comma separated list correspond to --name) " + "e.g. customGroup:customArtifact:customVersion"); + System.out.println(" --proxy-url [url] (Optional) proxy url. http(s)://host:port"); + System.out.println(" --proxy-user [user] (Optional) proxy user"); + System.out.println(" --proxy-password [password] (Optional) proxy password"); } public static void main(String [] args) throws IOException { @@ -195,6 +213,10 @@ public static void main(String [] args) throws IOException { String names = null; String artifacts = null; + URL proxyUrl = null; + String proxyUser = null; + String proxyPassword = null; + boolean all = false; for (int i = 0; i < args.length; i++) { String arg = args[i].toLowerCase(Locale.US); @@ -206,8 +228,7 @@ public static void main(String [] args) throws IOException { break; case "--all": case "-a": - installer.installAll(); - System.exit(0); + all = true; break; case "--name": case "-n": @@ -221,6 +242,15 @@ public static void main(String [] args) throws IOException { case "-v": Util.getVersion(); break; + case "--proxy-url": + proxyUrl = new URL(args[++i]); + break; + case "--proxy-user": + proxyUser = args[++i]; + break; + case "--proxy-password": + proxyPassword = args[++i]; + break; case "--help": case "-h": usage(); @@ -231,6 +261,15 @@ public static void main(String [] args) throws IOException { } } + if (proxyUrl != null) { + installer.setProxy(proxyUrl, proxyUser, proxyPassword); + } + + if (all) { + installer.installAll(); + System.exit(0); + } + if (names != null) { if (artifacts != null) { installer.install(names.split(","), artifacts.split(",")); From 13899fb05b997a2118673c844b837b7b23cd821d Mon Sep 17 00:00:00 2001 From: AhyoungRyu Date: Tue, 21 Jun 2016 23:40:31 -0700 Subject: [PATCH 15/22] Reorganize interpreter installation docs --- .../themes/zeppelin/_navigation.html | 3 +- docs/install/install.md | 214 ++++-------------- docs/manual/interpreterinstallation.md | 164 ++++++++++++++ 3 files changed, 205 insertions(+), 176 deletions(-) create mode 100644 docs/manual/interpreterinstallation.md diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index e93ca52b765..aa0af68e55a 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -42,6 +42,7 @@
  • Overview
  • Usage
  • +
  • Interpreter Installation
  • Dynamic Interpreter Loading
  • Interpreter Dependency Management
  • @@ -109,4 +110,4 @@ - + \ No newline at end of file diff --git a/docs/install/install.md b/docs/install/install.md index 1a057b0c1e0..cb8fd4aa391 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -19,115 +19,40 @@ limitations under the License. --> {% include JB/setup %} -# Quick Start -Welcome to your first trial to explore Apache Zeppelin! -This page will help you to get started and here is the list of topics covered. +## Zeppelin Installation +Welcome to your first trial to explore Zeppelin! -* [Installation](#installation) - * [Downloading Binary Package](#downloading-binary-package) - * [Building from Source](#building-from-source) -* [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) - * [Start Zeppelin](#start-zeppelin) - * [Stop Zeppelin](#stop-zeppelin) - * [(Optional) Start Apache Zeppelin with a service manager](#optional-start-apache-zeppelin-with-a-service-manager) -* [What is the next?](#what-is-the-next) -* [Apache Zeppelin Configuration](#apache-zeppelin-configuration) +In this documentation, we will explain how you can install Zeppelin from **Binary Package** or build from **Source** by yourself. Plus, you can see all of Zeppelin's configurations in the [Zeppelin Configuration](install.html#zeppelin-configuration) section below. -## Installation +### Install with Binary Package -Apache Zeppelin officially supports and is tested on next environments. +If you want to install Zeppelin with latest binary package, please visit [this page](http://zeppelin.apache.org/download.html). - - - - - - - - - - - - - -
    NameValue
    Oracle JDK1.7
    (set JAVA_HOME)
    OSMac OSX
    Ubuntu 14.X
    CentOS 6.X
    Windows 7 Pro SP1
    - -There are two options to install Apache Zeppelin on your machine. One is [downloading prebuild binary package](#downloading-binary-package) from the archive. -You can download not only the latest stable version but also the older one if you need. -The other option is [building from the source](#building-from-source). -Although it can be unstable somehow since it is on development status, you can explore newly added feature and change it as you want. - -### Downloading Binary Package - -If you want to install Apache Zeppelin with a stable binary package, please visit [Apache Zeppelin download Page](http://zeppelin.apache.org/download.html). -After unpacking, jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section. - -If you have downloaded `netinst` binary, [install additional interpreters](http://localhost:4000/install/install.html#install-interpreters) or simply run `./bin/install-interpreter.sh --all` before you jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section. - -### Building from Source -If you want to build from the source, the software below needs to be installed on your system. - - - - - - - - - - - - - - -
    NameValue
    Git
    Maven3.1.x or higher
    - -If you don't have it installed yet, please check [Before Build](https://github.com/apache/zeppelin/blob/master/README.md#before-build) section and follow step by step instructions from there. - -####1. Clone Apache Zeppelin repository +If you have downloaded `netinst` binary, [install additional interpreters](../manual/interpreterinstallation.html) before you start Zeppelin. Or simply run `./bin/install-interpreter.sh --all`. -``` -git clone https://github.com/apache/zeppelin.git -``` - -####2. Build source with options -Each interpreters requires different build options. For the further information about options, please see [Build](https://github.com/apache/zeppelin#build) section. - -``` -mvn clean package -DskipTests [Options] -``` +### Build from Zeppelin Source -Here are some examples with several options +You can also build Zeppelin from the source. -``` -# basic build -mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark +#### Prerequisites for build + * Java 1.7 + * Git + * Maven(3.1.x or higher) + * Node.js Package Manager -# spark-cassandra integration -mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests +If you don't have requirements prepared, please check instructions in [README.md](https://github.com/apache/zeppelin/blob/master/README.md) for the details. -# with CDH -mvn clean package -Pspark-1.5 -Dhadoop.version=2.6.0-cdh5.5.0 -Phadoop-2.6 -Pvendor-repo -DskipTests -# with MapR -mvn clean package -Pspark-1.5 -Pmapr50 -DskipTests -``` -For the further information about building with source, please see [README.md](https://github.com/apache/zeppelin/blob/master/README.md) in Zeppelin repository. +Maybe you need to configure individual interpreter. If so, please check **Interpreter** section in Zeppelin documentation. +[Spark Interpreter for Apache Zeppelin](../interpreter/spark.html) will be a good example. -## Starting Apache Zeppelin with Command Line +## Zeppelin Start / Stop #### Start Zeppelin ``` bin/zeppelin-daemon.sh start ``` - -If you are using Windows - -``` -bin\zeppelin.cmd -``` - After successful start, visit [http://localhost:8080](http://localhost:8080) with your web browser. #### Stop Zeppelin @@ -136,28 +61,21 @@ After successful start, visit [http://localhost:8080](http://localhost:8080) wit bin/zeppelin-daemon.sh stop ``` -#### (Optional) Start Apache Zeppelin with a service manager - -> **Note :** The below description was written based on Ubuntu Linux. +#### Start Zeppelin with a service manager such as upstart -Apache Zeppelin can be auto started as a service with an init script, such as services managed by **upstart**. +Zeppelin can auto start as a service with an init script, such as services managed by upstart. -The following is an example of upstart script to be saved as `/etc/init/zeppelin.conf` +The following is an example upstart script to be saved as `/etc/init/zeppelin.conf` +This example has been tested with Ubuntu Linux. This also allows the service to be managed with commands such as -``` -sudo service zeppelin start -sudo service zeppelin stop -sudo service zeppelin restart -``` - -Other service managers could use a similar approach with the `upstart` argument passed to the `zeppelin-daemon.sh` script. +`sudo service zeppelin start` +`sudo service zeppelin stop` +`sudo service zeppelin restart` -``` -bin/zeppelin-daemon.sh upstart -``` +Other service managers could use a similar approach with the `upstart` argument passed to the zeppelin-daemon.sh script: `bin/zeppelin-daemon.sh upstart` -**zeppelin.conf** +##### zeppelin.conf ``` description "zeppelin" @@ -177,16 +95,15 @@ chdir /usr/share/zeppelin exec bin/zeppelin-daemon.sh upstart ``` -## What is the next? -Congratulation on your successful Apache Zeppelin installation! Here are two next steps you might need. +#### Running on Windows - * For an in-depth overview of Apache Zeppelin UI, head to [Explore Apache Zeppelin UI](../quickstart/explorezeppelinui.html) - * After getting familiar with Apache Zeppelin UI, have fun with a short walk-through [Tutorial](../quickstart/tutorial.html) that uses Apache Spark backend - * If you need more configuration setting for Apache Zeppelin, jump to the next section: [Apache Zeppelin Configuration](#apache-zeppelin-configuration) +``` +bin\zeppelin.cmd +``` -## Apache Zeppelin Configuration +## Zeppelin Configuration -You can configure Apache Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority. +You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority. @@ -295,13 +212,13 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z - + - + @@ -313,13 +230,13 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z - + - + @@ -355,7 +272,7 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z - + @@ -376,13 +293,13 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z - + - + @@ -391,56 +308,3 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
    ZEPPELIN_NOTEBOOK_HOMESCREEN zeppelin.notebook.homescreen A notebook id displayed in Apache Zeppelin homescreen
    i.e. 2A94M5J1Z
    A notebook id displayed in Zeppelin homescreen
    i.e. 2A94M5J1Z
    ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE zeppelin.notebook.homescreen.hide falseThis value can be "true" when to hide the notebook id set by ZEPPELIN_NOTEBOOK_HOMESCREEN on the Apache Zeppelin homescreen.
    For the further information, please read Customize your Zeppelin homepage.
    This value can be "true" when to hide the notebook id set by ZEPPELIN_NOTEBOOK_HOMESCREEN on the Zeppelin homescreen.
    For the further information, please read Customize your Zeppelin homepage.
    ZEPPELIN_WAR_TEMPDIRZEPPELIN_NOTEBOOK_DIR zeppelin.notebook.dir notebookThe root directory where notebook directories are savedThe root directory where Zeppelin notebook directories are saved
    ZEPPELIN_NOTEBOOK_S3_BUCKET zeppelin.notebook.s3.bucket zeppelinS3 Bucket where notebook files will be savedS3 Bucket where Zeppelin notebook files will be saved
    ZEPPELIN_NOTEBOOK_S3_USERZEPPELIN_NOTEBOOK_AZURE_SHARE zeppelin.notebook.azure.share zeppelinShare where the notebook files will be savedShare where the Zeppelin notebook files will be saved
    ZEPPELIN_NOTEBOOK_AZURE_USERorg.apache.zeppelin.spark.SparkInterpreter,
    org.apache.zeppelin.spark.PySparkInterpreter,
    org.apache.zeppelin.spark.SparkSqlInterpreter,
    org.apache.zeppelin.spark.DepInterpreter,
    org.apache.zeppelin.markdown.Markdown,
    org.apache.zeppelin.shell.ShellInterpreter,
    ...
    Comma separated interpreter configurations [Class]
    The first interpreter will be a default value.
    It means only the first interpreter in this list can be available without %interpreter_name annotation in notebook paragraph.
    Comma separated interpreter configurations [Class]
    The first interpreter will be a default value.
    It means only the first interpreter in this list can be available without %interpreter_name annotation in Zeppelin notebook paragraph.
    ZEPPELIN_INTERPRETER_DIR zeppelin.interpreter.dir interpreterInterpreter directoryZeppelin interpreter directory
    ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZESize in characters of the maximum text message to be received by websocket.
    - -## Install interpreters - -You can install additional [interpreters](../manual/interpreters.html) using `bin/install-interpreter.sh` command. - -### Community managed interpreters - -Informations of community managed interpreters are listed in `conf/interpreter-list` file. `bin/install-interpreter.sh` command will read this file to install interpreters. - -##### Install all community managed interpreters - -``` -./bin/install-interpreter.sh --all -``` - -##### Install specific interpreters - -``` -./bin/install-interpreter.sh --name md,shell,jdbc,python -``` - -You can get full list of community managed interpreters by running - -``` -./bin/install-interpreter.sh --list -``` - -Once you have installed interpreter, restart Zeppelin. And then you'll need [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [binding it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). - - -### 3rd party interpreters - -`./bin/install-interpreter.sh` command can install 3rd party interpreters available in maven repository. - -##### Install 3rd party interpreters - -``` -./bin/install-interpreter.sh --name interpreter1 --repository groupId1:artifact1:version1 -``` - -The command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory. - -Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](install.html#zeppelin-configuration). -And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [binding it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). - - -##### Install multiple 3rd party interpreters at once - -`--name` and `--repository` argument receives comma separated list - -``` -./bin/install-interpreter.sh --name interpreter1,interpreter2 --repository groupId1:artifact1:version1,groupId2:artifact2:version2 -``` diff --git a/docs/manual/interpreterinstallation.md b/docs/manual/interpreterinstallation.md new file mode 100644 index 00000000000..38745d91dd7 --- /dev/null +++ b/docs/manual/interpreterinstallation.md @@ -0,0 +1,164 @@ +--- +layout: page +title: "Getting Started" +description: "" +group: install +--- + +{% include JB/setup %} + +# Interpreter Installation + +Apache Zeppelin provides **Interpreter Installation** mechanism for whom downloaded Zeppelin `netinst` binary package, or just want to install another 3rd party interpreters. + +## Community managed interpreters +Apache Zeppelin provides several interpreters as [community managed interpreters](#available-community-managed-interpreters). +If you downloaded `netinst` binary package, you need to install by using below commands. + +#### Install all community managed interpreters + +``` +./bin/install-interpreter.sh --all +``` + +#### Install specific interpreters + +``` +./bin/install-interpreter.sh --name md,shell,jdbc,python +``` + +You can get full list of community managed interpreters by running + +``` +./bin/install-interpreter.sh --list +``` + +Once you have installed interpreter, restart Zeppelin. And then you'll need to [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). + + +## 3rd party interpreters + +You can also install 3rd party interpreters located in the maven repository by using below commands. + +#### Install 3rd party interpreters + +``` +./bin/install-interpreter.sh --name interpreter1 --repository groupId1:artifact1:version1 +``` + +The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory. + +Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/install.html#zeppelin-configuration). +And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). + + +#### Install multiple 3rd party interpreters at once + +``` +./bin/install-interpreter.sh --name interpreter1,interpreter2 --repository groupId1:artifact1:version1,groupId2:artifact2:version2 +``` + +`--name` and `--repository` argument will recieve comma separated list. + +## Available community managed interpreters + +You can also find the below community managed interpreter list in `conf/interpreter-list` file. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameMaven ArtifactDescription
    alluxioorg.apache.zeppelin.zeppelin-alluxio:0.6.0Alluxio interpreter
    angularorg.apache.zeppelin.zeppelin-angular:0.6.0HTML and AngularJS view rendering
    cassandraorg.apache.zeppelin.zeppelin-cassandra:0.6.0HTML and AngularJS view rendering
    elasticsearchorg.apache.zeppelin.zeppelin-elasticsearch:0.6.0Elasticsearch interpreter
    fileorg.apache.zeppelin:zeppelin-file:0.6.0HDFS file interpreter
    flinkorg.apache.zeppelin:zeppelin-flink:0.6.0Flink interpreter
    hbaseorg.apache.zeppelin:zeppelin-hbase:0.6.0Hbase interpreter
    igniteorg.apache.zeppelin:zeppelin-ignite:0.6.0Ignite interpreter
    jdbcorg.apache.zeppelin:zeppelin-jdbc:0.6.0Jdbc interpreter
    kylinorg.apache.zeppelin:zeppelin-kylin:0.6.0Kylin interpreter
    lensorg.apache.zeppelin:zeppelin-lens:0.6.0Lens interpreter
    livyorg.apache.zeppelin:zeppelin-livy:0.6.0Livy interpreter
    mdorg.apache.zeppelin:zeppelin-markdown:0.6.0Markdown support
    postgresqlorg.apache.zeppelin:zeppelin-postgresql:0.6.0Postgresql interpreter
    pythonorg.apache.zeppelin:zeppelin-python:0.6.0Python interpreter
    shellorg.apache.zeppelin:zeppelin-shell:0.6.0Shell command
    From 5d0a9718c9126b9039aae797378b372bb5c625d8 Mon Sep 17 00:00:00 2001 From: AhyoungRyu Date: Tue, 21 Jun 2016 23:44:04 -0700 Subject: [PATCH 16/22] Revert install.md to latest version --- docs/install/install.md | 161 ++++++++++++++++++++++++++++++---------- 1 file changed, 122 insertions(+), 39 deletions(-) diff --git a/docs/install/install.md b/docs/install/install.md index cb8fd4aa391..179f99859f8 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -19,40 +19,115 @@ limitations under the License. --> {% include JB/setup %} -## Zeppelin Installation -Welcome to your first trial to explore Zeppelin! +# Quick Start +Welcome to your first trial to explore Apache Zeppelin! +This page will help you to get started and here is the list of topics covered. -In this documentation, we will explain how you can install Zeppelin from **Binary Package** or build from **Source** by yourself. Plus, you can see all of Zeppelin's configurations in the [Zeppelin Configuration](install.html#zeppelin-configuration) section below. +* [Installation](#installation) + * [Downloading Binary Package](#downloading-binary-package) + * [Building from Source](#building-from-source) +* [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) + * [Start Zeppelin](#start-zeppelin) + * [Stop Zeppelin](#stop-zeppelin) + * [(Optional) Start Apache Zeppelin with a service manager](#optional-start-apache-zeppelin-with-a-service-manager) +* [What is the next?](#what-is-the-next) +* [Apache Zeppelin Configuration](#apache-zeppelin-configuration) -### Install with Binary Package +## Installation -If you want to install Zeppelin with latest binary package, please visit [this page](http://zeppelin.apache.org/download.html). +Apache Zeppelin officially supports and is tested on next environments. + + + + + + + + + + + + + +
    NameValue
    Oracle JDK1.7
    (set JAVA_HOME)
    OSMac OSX
    Ubuntu 14.X
    CentOS 6.X
    Windows 7 Pro SP1
    + +There are two options to install Apache Zeppelin on your machine. One is [downloading prebuild binary package](#downloading-binary-package) from the archive. +You can download not only the latest stable version but also the older one if you need. +The other option is [building from the source](#building-from-source). +Although it can be unstable somehow since it is on development status, you can explore newly added feature and change it as you want. + +### Downloading Binary Package + +If you want to install Apache Zeppelin with a stable binary package, please visit [Apache Zeppelin download Page](http://zeppelin.apache.org/download.html). If you have downloaded `netinst` binary, [install additional interpreters](../manual/interpreterinstallation.html) before you start Zeppelin. Or simply run `./bin/install-interpreter.sh --all`. -### Build from Zeppelin Source +After unpacking, jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section. -You can also build Zeppelin from the source. +### Building from Source +If you want to build from the source, the software below needs to be installed on your system. -#### Prerequisites for build - * Java 1.7 - * Git - * Maven(3.1.x or higher) - * Node.js Package Manager + + + + + + + + + + + + + +
    NameValue
    Git
    Maven3.1.x or higher
    + +If you don't have it installed yet, please check [Before Build](https://github.com/apache/zeppelin/blob/master/README.md#before-build) section and follow step by step instructions from there. -If you don't have requirements prepared, please check instructions in [README.md](https://github.com/apache/zeppelin/blob/master/README.md) for the details. +####1. Clone Apache Zeppelin repository +``` +git clone https://github.com/apache/zeppelin.git +``` +####2. Build source with options +Each interpreters requires different build options. For the further information about options, please see [Build](https://github.com/apache/zeppelin#build) section. -Maybe you need to configure individual interpreter. If so, please check **Interpreter** section in Zeppelin documentation. -[Spark Interpreter for Apache Zeppelin](../interpreter/spark.html) will be a good example. +``` +mvn clean package -DskipTests [Options] +``` -## Zeppelin Start / Stop +Here are some examples with several options + +``` +# basic build +mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark + +# spark-cassandra integration +mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests + +# with CDH +mvn clean package -Pspark-1.5 -Dhadoop.version=2.6.0-cdh5.5.0 -Phadoop-2.6 -Pvendor-repo -DskipTests + +# with MapR +mvn clean package -Pspark-1.5 -Pmapr50 -DskipTests +``` + +For the further information about building with source, please see [README.md](https://github.com/apache/zeppelin/blob/master/README.md) in Zeppelin repository. + +## Starting Apache Zeppelin with Command Line #### Start Zeppelin ``` bin/zeppelin-daemon.sh start ``` + +If you are using Windows + +``` +bin\zeppelin.cmd +``` + After successful start, visit [http://localhost:8080](http://localhost:8080) with your web browser. #### Stop Zeppelin @@ -61,21 +136,28 @@ After successful start, visit [http://localhost:8080](http://localhost:8080) wit bin/zeppelin-daemon.sh stop ``` -#### Start Zeppelin with a service manager such as upstart +#### (Optional) Start Apache Zeppelin with a service manager + +> **Note :** The below description was written based on Ubuntu Linux. -Zeppelin can auto start as a service with an init script, such as services managed by upstart. +Apache Zeppelin can be auto started as a service with an init script, such as services managed by **upstart**. -The following is an example upstart script to be saved as `/etc/init/zeppelin.conf` -This example has been tested with Ubuntu Linux. +The following is an example of upstart script to be saved as `/etc/init/zeppelin.conf` This also allows the service to be managed with commands such as -`sudo service zeppelin start` -`sudo service zeppelin stop` -`sudo service zeppelin restart` +``` +sudo service zeppelin start +sudo service zeppelin stop +sudo service zeppelin restart +``` + +Other service managers could use a similar approach with the `upstart` argument passed to the `zeppelin-daemon.sh` script. -Other service managers could use a similar approach with the `upstart` argument passed to the zeppelin-daemon.sh script: `bin/zeppelin-daemon.sh upstart` +``` +bin/zeppelin-daemon.sh upstart +``` -##### zeppelin.conf +**zeppelin.conf** ``` description "zeppelin" @@ -95,15 +177,16 @@ chdir /usr/share/zeppelin exec bin/zeppelin-daemon.sh upstart ``` -#### Running on Windows +## What is the next? +Congratulation on your successful Apache Zeppelin installation! Here are two next steps you might need. -``` -bin\zeppelin.cmd -``` + * For an in-depth overview of Apache Zeppelin UI, head to [Explore Apache Zeppelin UI](../quickstart/explorezeppelinui.html) + * After getting familiar with Apache Zeppelin UI, have fun with a short walk-through [Tutorial](../quickstart/tutorial.html) that uses Apache Spark backend + * If you need more configuration setting for Apache Zeppelin, jump to the next section: [Apache Zeppelin Configuration](#apache-zeppelin-configuration) -## Zeppelin Configuration +## Apache Zeppelin Configuration -You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority. +You can configure Apache Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority. @@ -212,13 +295,13 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin - + - + @@ -230,13 +313,13 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin - + - + @@ -272,7 +355,7 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin - + @@ -293,13 +376,13 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin - + - + @@ -307,4 +390,4 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin -
    ZEPPELIN_NOTEBOOK_HOMESCREEN zeppelin.notebook.homescreen A notebook id displayed in Zeppelin homescreen
    i.e. 2A94M5J1Z
    A notebook id displayed in Apache Zeppelin homescreen
    i.e. 2A94M5J1Z
    ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE zeppelin.notebook.homescreen.hide falseThis value can be "true" when to hide the notebook id set by ZEPPELIN_NOTEBOOK_HOMESCREEN on the Zeppelin homescreen.
    For the further information, please read Customize your Zeppelin homepage.
    This value can be "true" when to hide the notebook id set by ZEPPELIN_NOTEBOOK_HOMESCREEN on the Apache Zeppelin homescreen.
    For the further information, please read Customize your Zeppelin homepage.
    ZEPPELIN_WAR_TEMPDIRZEPPELIN_NOTEBOOK_DIR zeppelin.notebook.dir notebookThe root directory where Zeppelin notebook directories are savedThe root directory where notebook directories are saved
    ZEPPELIN_NOTEBOOK_S3_BUCKET zeppelin.notebook.s3.bucket zeppelinS3 Bucket where Zeppelin notebook files will be savedS3 Bucket where notebook files will be saved
    ZEPPELIN_NOTEBOOK_S3_USERZEPPELIN_NOTEBOOK_AZURE_SHARE zeppelin.notebook.azure.share zeppelinShare where the Zeppelin notebook files will be savedShare where the notebook files will be saved
    ZEPPELIN_NOTEBOOK_AZURE_USERorg.apache.zeppelin.spark.SparkInterpreter,
    org.apache.zeppelin.spark.PySparkInterpreter,
    org.apache.zeppelin.spark.SparkSqlInterpreter,
    org.apache.zeppelin.spark.DepInterpreter,
    org.apache.zeppelin.markdown.Markdown,
    org.apache.zeppelin.shell.ShellInterpreter,
    ...
    Comma separated interpreter configurations [Class]
    The first interpreter will be a default value.
    It means only the first interpreter in this list can be available without %interpreter_name annotation in Zeppelin notebook paragraph.
    Comma separated interpreter configurations [Class]
    The first interpreter will be a default value.
    It means only the first interpreter in this list can be available without %interpreter_name annotation in notebook paragraph.
    ZEPPELIN_INTERPRETER_DIR zeppelin.interpreter.dir interpreterZeppelin interpreter directoryInterpreter directory
    ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE1024000 Size in characters of the maximum text message to be received by websocket.
    + \ No newline at end of file From 03c664eb9bcaedd5061dad3bcfea252b1aa2be5d Mon Sep 17 00:00:00 2001 From: AhyoungRyu Date: Tue, 21 Jun 2016 23:45:36 -0700 Subject: [PATCH 17/22] Add a new line --- docs/install/install.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/install/install.md b/docs/install/install.md index 179f99859f8..55741dcf1cb 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -60,6 +60,7 @@ Although it can be unstable somehow since it is on development status, you can e ### Downloading Binary Package If you want to install Apache Zeppelin with a stable binary package, please visit [Apache Zeppelin download Page](http://zeppelin.apache.org/download.html). + If you have downloaded `netinst` binary, [install additional interpreters](../manual/interpreterinstallation.html) before you start Zeppelin. Or simply run `./bin/install-interpreter.sh --all`. After unpacking, jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section. From 13f2d0457de0d5826e66e0485535cdb166779710 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 21 Jun 2016 23:55:41 -0700 Subject: [PATCH 18/22] generate netinst package --- dev/create_release.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/create_release.sh b/dev/create_release.sh index f1bba0dded6..5b3727853b8 100755 --- a/dev/create_release.sh +++ b/dev/create_release.sh @@ -103,6 +103,7 @@ function make_binary_release() { git_clone make_source_package make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" +make_binary_release netinst "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -pl '!alluxio,!angular,!cassandra,!elasticsearch,!file,!flink,!hbase,!ignite,!jdbc,!kylin,!lens,!livy,!markdown,!postgresql,!python,!shell" # remove non release files and dirs rm -rf "${WORKING_DIR}/zeppelin" From 0eedd2aca57a68111b5ed2ac11a651fe8546e86f Mon Sep 17 00:00:00 2001 From: AhyoungRyu Date: Wed, 22 Jun 2016 23:54:17 -0700 Subject: [PATCH 19/22] Address @minahlee feedback --- .../themes/zeppelin/_navigation.html | 6 +- docs/manual/interpreterinstallation.md | 164 ++++++++++++++++++ 2 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 docs/manual/interpreterinstallation.md diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index 40b9fb6024b..498a2d36782 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -21,8 +21,8 @@
  • What is Apache Zeppelin ?
  • Getting Started
  • -
  • Quick Start
  • -
  • Configuration
  • +
  • Install
  • +
  • Configuration
  • Explore Zeppelin UI
  • Tutorial
  • @@ -42,6 +42,7 @@
  • Overview
  • Usage
  • +
  • Interpreter Installation
  • Dynamic Interpreter Loading
  • Interpreter Dependency Management
  • @@ -110,3 +111,4 @@ + \ No newline at end of file diff --git a/docs/manual/interpreterinstallation.md b/docs/manual/interpreterinstallation.md new file mode 100644 index 00000000000..f3d83a9db94 --- /dev/null +++ b/docs/manual/interpreterinstallation.md @@ -0,0 +1,164 @@ +--- +layout: page +title: "Interpreter Installation" +description: "" +group: manual +--- + +{% include JB/setup %} + +# Interpreter Installation + +Apache Zeppelin provides **Interpreter Installation** mechanism for whom downloaded Zeppelin `netinst` binary package, or just want to install another 3rd party interpreters. + +## Community managed interpreters +Apache Zeppelin provides several interpreters as [community managed interpreters](#available-community-managed-interpreters). +If you downloaded `netinst` binary package, you need to install by using below commands. + +#### Install all community managed interpreters + +``` +./bin/install-interpreter.sh --all +``` + +#### Install specific interpreters + +``` +./bin/install-interpreter.sh --name md,shell,jdbc,python +``` + +You can get full list of community managed interpreters by running + +``` +./bin/install-interpreter.sh --list +``` + +Once you have installed interpreters, you need to restart Zeppelin. And then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). + + +## 3rd party interpreters + +You can also install 3rd party interpreters located in the maven repository by using below commands. + +#### Install 3rd party interpreters + +``` +./bin/install-interpreter.sh --name interpreter1 --repository groupId1:artifact1:version1 +``` + +The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory. + +Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/install.html#apache-zeppelin-configuration). +And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). + + +#### Install multiple 3rd party interpreters at once + +``` +./bin/install-interpreter.sh --name interpreter1,interpreter2 --repository groupId1:artifact1:version1,groupId2:artifact2:version2 +``` + +`--name` and `--repository` arguments will recieve comma separated list. + +## Available community managed interpreters + +You can also find the below community managed interpreter list in `conf/interpreter-list` file. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameMaven ArtifactDescription
    alluxioorg.apache.zeppelin.zeppelin-alluxio:0.6.0Alluxio interpreter
    angularorg.apache.zeppelin.zeppelin-angular:0.6.0HTML and AngularJS view rendering
    cassandraorg.apache.zeppelin.zeppelin-cassandra:0.6.0Cassandra interpreter
    elasticsearchorg.apache.zeppelin.zeppelin-elasticsearch:0.6.0Elasticsearch interpreter
    fileorg.apache.zeppelin:zeppelin-file:0.6.0HDFS file interpreter
    flinkorg.apache.zeppelin:zeppelin-flink:0.6.0Flink interpreter
    hbaseorg.apache.zeppelin:zeppelin-hbase:0.6.0Hbase interpreter
    igniteorg.apache.zeppelin:zeppelin-ignite:0.6.0Ignite interpreter
    jdbcorg.apache.zeppelin:zeppelin-jdbc:0.6.0Jdbc interpreter
    kylinorg.apache.zeppelin:zeppelin-kylin:0.6.0Kylin interpreter
    lensorg.apache.zeppelin:zeppelin-lens:0.6.0Lens interpreter
    livyorg.apache.zeppelin:zeppelin-livy:0.6.0Livy interpreter
    mdorg.apache.zeppelin:zeppelin-markdown:0.6.0Markdown support
    postgresqlorg.apache.zeppelin:zeppelin-postgresql:0.6.0Postgresql interpreter
    pythonorg.apache.zeppelin:zeppelin-python:0.6.0Python interpreter
    shellorg.apache.zeppelin:zeppelin-shell:0.6.0Shell command
    From 7e749ad84d9dbb56578aeda1561b310fe2512a8c Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 23 Jun 2016 09:04:40 -0700 Subject: [PATCH 20/22] Address mina's comment --- conf/interpreter-list | 8 ++++---- docs/_includes/themes/zeppelin/_navigation.html | 4 ++-- docs/manual/interpreterinstallation.md | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/conf/interpreter-list b/conf/interpreter-list index 94940716793..a2f64269633 100644 --- a/conf/interpreter-list +++ b/conf/interpreter-list @@ -17,10 +17,10 @@ # # [name] [maven artifact] [description] -alluxio org.apache.zeppelin.zeppelin-alluxio:0.6.0 Alluxio interpreter -angular org.apache.zeppelin.zeppelin-angular:0.6.0 HTML and AngularJS view rendering -cassandra org.apache.zeppelin.zeppelin-cassandra:0.6.0 Cassandra interpreter -elasticsearch org.apache.zeppelin.zeppelin-elasticsearch:0.6.0 Elasticsearch interpreter +alluxio org.apache.zeppelin:zeppelin-alluxio:0.6.0 Alluxio interpreter +angular org.apache.zeppelin:zeppelin-angular:0.6.0 HTML and AngularJS view rendering +cassandra org.apache.zeppelin:zeppelin-cassandra:0.6.0 Cassandra interpreter +elasticsearch org.apache.zeppelin:zeppelin-elasticsearch:0.6.0 Elasticsearch interpreter file org.apache.zeppelin:zeppelin-file:0.6.0 HDFS file interpreter flink org.apache.zeppelin:zeppelin-flink:0.6.0 Flink interpreter hbase org.apache.zeppelin:zeppelin-hbase:0.6.0 Hbase interpreter diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index aa0af68e55a..c56a6295588 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -22,7 +22,7 @@
  • Getting Started
  • Install
  • -
  • Configuration
  • +
  • Configuration
  • Explore Zeppelin UI
  • Tutorial
  • @@ -110,4 +110,4 @@ - \ No newline at end of file + diff --git a/docs/manual/interpreterinstallation.md b/docs/manual/interpreterinstallation.md index 38745d91dd7..499aea507cd 100644 --- a/docs/manual/interpreterinstallation.md +++ b/docs/manual/interpreterinstallation.md @@ -60,7 +60,7 @@ You can also install 3rd party interpreters located in the maven repository by u The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory. -Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/install.html#zeppelin-configuration). +Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/install.html#apache-zeppelin-configuration). And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). @@ -94,7 +94,7 @@ You can also find the below community managed interpreter list in `conf/interpre cassandra org.apache.zeppelin.zeppelin-cassandra:0.6.0 - HTML and AngularJS view rendering + Cassandra interpreter elasticsearch From 049bc89dabc32be1c211febddce2646bfc0f214f Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 23 Jun 2016 11:42:20 -0700 Subject: [PATCH 21/22] Update docs --- docs/manual/interpreterinstallation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/manual/interpreterinstallation.md b/docs/manual/interpreterinstallation.md index f3d83a9db94..4dc27529861 100644 --- a/docs/manual/interpreterinstallation.md +++ b/docs/manual/interpreterinstallation.md @@ -83,22 +83,22 @@ You can also find the below community managed interpreter list in `conf/interpre alluxio - org.apache.zeppelin.zeppelin-alluxio:0.6.0 + org.apache.zeppelin:zeppelin-alluxio:0.6.0 Alluxio interpreter angular - org.apache.zeppelin.zeppelin-angular:0.6.0 + org.apache.zeppelin:zeppelin-angular:0.6.0 HTML and AngularJS view rendering cassandra - org.apache.zeppelin.zeppelin-cassandra:0.6.0 + org.apache.zeppelin:zeppelin-cassandra:0.6.0 Cassandra interpreter elasticsearch - org.apache.zeppelin.zeppelin-elasticsearch:0.6.0 + org.apache.zeppelin:zeppelin-elasticsearch:0.6.0 Elasticsearch interpreter From f81d16e37976506c699d1859b7e5b1e7ec78e9e5 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 23 Jun 2016 15:04:43 -0700 Subject: [PATCH 22/22] address mina's comment --- dev/create_release.sh | 2 +- docs/manual/interpreterinstallation.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/create_release.sh b/dev/create_release.sh index 5b3727853b8..2363d904bcc 100755 --- a/dev/create_release.sh +++ b/dev/create_release.sh @@ -103,7 +103,7 @@ function make_binary_release() { git_clone make_source_package make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" -make_binary_release netinst "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -pl '!alluxio,!angular,!cassandra,!elasticsearch,!file,!flink,!hbase,!ignite,!jdbc,!kylin,!lens,!livy,!markdown,!postgresql,!python,!shell" +make_binary_release netinst "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -pl !alluxio,!angular,!cassandra,!elasticsearch,!file,!flink,!hbase,!ignite,!jdbc,!kylin,!lens,!livy,!markdown,!postgresql,!python,!shell" # remove non release files and dirs rm -rf "${WORKING_DIR}/zeppelin" diff --git a/docs/manual/interpreterinstallation.md b/docs/manual/interpreterinstallation.md index 4dc27529861..d522e620e53 100644 --- a/docs/manual/interpreterinstallation.md +++ b/docs/manual/interpreterinstallation.md @@ -55,7 +55,7 @@ You can also install 3rd party interpreters located in the maven repository by u #### Install 3rd party interpreters ``` -./bin/install-interpreter.sh --name interpreter1 --repository groupId1:artifact1:version1 +./bin/install-interpreter.sh --name interpreter1 --artifact groupId1:artifact1:version1 ``` The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory. @@ -67,10 +67,10 @@ And then restart Zeppelin, [create interpreter setting](../manual/interpreters.h #### Install multiple 3rd party interpreters at once ``` -./bin/install-interpreter.sh --name interpreter1,interpreter2 --repository groupId1:artifact1:version1,groupId2:artifact2:version2 +./bin/install-interpreter.sh --name interpreter1,interpreter2 --artifact groupId1:artifact1:version1,groupId2:artifact2:version2 ``` -`--name` and `--repository` arguments will recieve comma separated list. +`--name` and `--artifact` arguments will recieve comma separated list. ## Available community managed interpreters