diff --git a/bin/interpreter.sh b/bin/interpreter.sh index b5603c8ed46..b5a6d12346a 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -21,10 +21,10 @@ bin=$(cd "${bin}">/dev/null; pwd) function usage() { - echo "usage) $0 -p -d " + echo "usage) $0 -p -d -l " } -while getopts "hp:d:" o; do +while getopts "hp:d:l:" o; do case ${o} in h) usage @@ -36,6 +36,9 @@ while getopts "hp:d:" o; do p) PORT=${OPTARG} ;; + l) + LOCAL_INTERPRETER_REPO=${OPTARG} + ;; esac done @@ -128,6 +131,8 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then fi fi +addJarInDir "${LOCAL_INTERPRETER_REPO}" + CLASSPATH+=":${ZEPPELIN_CLASSPATH}" if [[ -n "${SPARK_SUBMIT}" ]]; then diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index b0b1a5b6e50..2fa53118bc4 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -29,10 +29,11 @@ # export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved # export ZEPPELIN_NOTEBOOK_HOMESCREEN # Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z # export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE # hide homescreen notebook from list when this value set to "true". default "false" -# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved -# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json +# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved +# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json # export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default. # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. +# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading #### Spark interpreter configuration #### diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 9ca740d139d..1d085ec24ae 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -103,6 +103,12 @@ Interpreter implementation base directory + + zeppelin.interpreter.localRepo + local-repo + Local repository for interpreter's additional dependency loading + + zeppelin.interpreters org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.tachyon.TachyonInterpreter diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index d0581b14549..7a7336589c4 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -37,7 +37,6 @@ Interpreter
  • diff --git a/docs/_includes/themes/zeppelin/default.html b/docs/_includes/themes/zeppelin/default.html index 3940414d5c7..9f1d6d58ca1 100644 --- a/docs/_includes/themes/zeppelin/default.html +++ b/docs/_includes/themes/zeppelin/default.html @@ -11,9 +11,11 @@ + + diff --git a/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png b/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png new file mode 100644 index 00000000000..d992e23be44 Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo1.png differ diff --git a/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png b/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png new file mode 100644 index 00000000000..2f117c10bd2 Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/interpreter-add-repo2.png differ diff --git a/docs/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png b/docs/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png new file mode 100644 index 00000000000..749d3910193 Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/interpreter-dependency-loading.png differ diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index 6141619f007..83eb3550ed1 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -80,9 +80,58 @@ SparkContext, SQLContext, ZeppelinContext are automatically created and exposed ## Dependency Management -There are two ways to load external library in spark interpreter. First is using Zeppelin's `%dep` interpreter and second is loading Spark properties. +There are two ways to load external library in spark interpreter. First is using Interpreter setting menu and second is loading Spark properties. + +### 1. Setting Dependencies via Interpreter Setting +Please see [Dependency Management](../manual/dependencymanagement.html) for the details. + +### 2. Loading Spark Properties +Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in conf/zeppelin-env.sh. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properites that user can set to distribute libraries are: + + + + + + + + + + + + + + + + + + + + + + + + + + +
    spark-defaults.confSPARK_SUBMIT_OPTIONSApplicable InterpreterDescription
    spark.jars--jars%sparkComma-separated list of local jars to include on the driver and executor classpaths.
    spark.jars.packages--packages%sparkComma-separated list of maven coordinates of jars to include on the driver and executor classpaths. Will search the local maven repo, then maven central and any additional remote repositories given by --repositories. The format for the coordinates should be groupId:artifactId:version.
    spark.files--files%pysparkComma-separated list of files to be placed in the working directory of each executor.
    +> Note that adding jar to pyspark is only availabe via `%dep` interpreter at the moment. + +Here are few examples: + +* SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh + + export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg" + +* SPARK_HOME/conf/spark-defaults.conf + + spark.jars /path/mylib1.jar,/path/mylib2.jar + spark.jars.packages com.databricks:spark-csv_2.10:1.2.0 + spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip + +### 3. Dynamic Dependency Loading via %dep interpreter +> Note: `%dep` interpreter is deprecated since v0.6.0-incubating. +`%dep` interpreter load libraries to `%spark` and `%pyspark` but not to `%spark.sql` interpreter so we recommend you to use first option instead. -### 1. Dynamic Dependency Loading via %dep interpreter When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using `%dep` interpreter. * Load libraries recursively from Maven repository @@ -129,49 +178,6 @@ z.load("groupId:artifactId:version").exclude("groupId:*") z.load("groupId:artifactId:version").local() ``` -### 2. Loading Spark Properties -Once `SPARK_HOME` is set in `conf/zeppelin-env.sh`, Zeppelin uses `spark-submit` as spark interpreter runner. `spark-submit` supports two ways to load configurations. The first is command line options such as --master and Zeppelin can pass these options to `spark-submit` by exporting `SPARK_SUBMIT_OPTIONS` in conf/zeppelin-env.sh. Second is reading configuration options from `SPARK_HOME/conf/spark-defaults.conf`. Spark properites that user can set to distribute libraries are: - - - - - - - - - - - - - - - - - - - - - - - - - - -
    spark-defaults.confSPARK_SUBMIT_OPTIONSApplicable InterpreterDescription
    spark.jars--jars%sparkComma-separated list of local jars to include on the driver and executor classpaths.
    spark.jars.packages--packages%sparkComma-separated list of maven coordinates of jars to include on the driver and executor classpaths. Will search the local maven repo, then maven central and any additional remote repositories given by --repositories. The format for the coordinates should be groupId:artifactId:version.
    spark.files--files%pysparkComma-separated list of files to be placed in the working directory of each executor.
    -> Note that adding jar to pyspark is only availabe via `%dep` interpreter at the moment. - -Here are few examples: - -* SPARK\_SUBMIT\_OPTIONS in conf/zeppelin-env.sh - - export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0 --jars /path/mylib1.jar,/path/mylib2.jar --files /path/mylib1.py,/path/mylib2.zip,/path/mylib3.egg" - -* SPARK_HOME/conf/spark-defaults.conf - - spark.jars /path/mylib1.jar,/path/mylib2.jar - spark.jars.packages com.databricks:spark-csv_2.10:1.2.0 - spark.files /path/mylib1.py,/path/mylib2.egg,/path/mylib3.zip - ## ZeppelinContext Zeppelin automatically injects ZeppelinContext as variable 'z' in your scala/python environment. ZeppelinContext provides some additional functions and utility. diff --git a/docs/manual/dependencymanagement.md b/docs/manual/dependencymanagement.md new file mode 100644 index 00000000000..612901e7a27 --- /dev/null +++ b/docs/manual/dependencymanagement.md @@ -0,0 +1,74 @@ +--- +layout: page +title: "Dependnecy Management" +description: "" +group: manual +--- + +{% include JB/setup %} + +## Dependency Management for Interpreter + +You can include external libraries to interpreter by setting dependencies in interpreter menu. + +When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs in this menu. + + * Load libraries recursively from Maven repository + * Load libraries from local filesystem + * Add additional maven repository + * Automatically add libraries to SparkCluster + +
    +
    +
    + + + +
    +
    + Load Dependencies to Interpreter +

    +
      +
    1. Click 'Interpreter' menu in navigation bar.
      2. +
    2. Click 'edit' button of the interpreter which you want to load dependencies to.
      3. +
    3. Fill artifact and exclude field to your needs. + You can enter not only groupId:artifactId:version but also local file in artifact field.
      4. +
    4. Press 'Save' to restart the interpreter with loaded libraries.
      5. +
    +
    +
    +
    +
    +
    + + + + + + +
    +
    + Add repository for dependency resolving +

    +
      +
    1. Press icon in 'Interpreter' menu on the top right side. + It will show you available repository lists.
      2. +
    2. If you need to resolve dependencies from other than central maven repository or + local ~/.m2 repository, hit icon next to repository lists.
      3. +
    3. Fill out the form and click 'Add' button, then you will be able to see that new repository is added.
      4. +
    +
    +
    + diff --git a/docs/rest-api/rest-interpreter.md b/docs/rest-api/rest-interpreter.md index 0db84706222..a2053eb4231 100644 --- a/docs/rest-api/rest-interpreter.md +++ b/docs/rest-api/rest-interpreter.md @@ -151,7 +151,8 @@ limitations under the License. "class": "org.apache.zeppelin.markdown.Markdown", "name": "md" } - ] + ], + "dependencies": [] }, { "id": "2AY6GV7Q3", @@ -170,6 +171,11 @@ limitations under the License. "class": "org.apache.zeppelin.spark.SparkSqlInterpreter", "name": "sql" } + ], + "dependencies": [ + { + "groupArtifactVersion": "com.databricks:spark-csv_2.10:1.3.0" + } ] } ] @@ -219,6 +225,12 @@ limitations under the License. "class": "org.apache.zeppelin.markdown.Markdown", "name": "md" } + ], + "dependencies": [ + { + "groupArtifactVersion": "groupId:artifactId:version", + "exclusions": "groupId:artifactId" + } ] } @@ -243,6 +255,12 @@ limitations under the License. "class": "org.apache.zeppelin.markdown.Markdown", "name": "md" } + ], + "dependencies": [ + { + "groupArtifactVersion": "groupId:artifactId:version", + "exclusions": "groupId:artifactId" + } ] } } @@ -292,6 +310,12 @@ limitations under the License. "class": "org.apache.zeppelin.markdown.Markdown", "name": "md" } + ], + "dependencies": [ + { + "groupArtifactVersion": "groupId:artifactId:version", + "exclusions": "groupId:artifactId" + } ] } @@ -316,6 +340,12 @@ limitations under the License. "class": "org.apache.zeppelin.markdown.Markdown", "name": "md" } + ], + "dependencies": [ + { + "groupArtifactVersion": "groupId:artifactId:version", + "exclusions": "groupId:artifactId" + } ] } } @@ -391,3 +421,75 @@ limitations under the License. + +
    +### 6. Add repository for dependency resolving + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Add new repository for dependency loader
    DescriptionThis ```POST``` method adds new repository.
    URL```http://[zeppelin-server]:[zeppelin-port]/api/interpreter/repository```
    Success code201
    Fail code 500
    Sample JSON input + 
    +{
+  "id": "securecentral",
+  "url": "https://repo1.maven.org/maven2",
+  "snapshot": false
+}
+
    +
    Sample JSON response + {"status":"OK"} +
    + +
    +### 7. Delete repository for dependency resolving + + + + + + + + + + + + + + + + + + + + + + + +
    Delete repository for dependency loader
    DescriptionThis ```DELETE``` method delete repository with given id.
    URL```http://[zeppelin-server]:[zeppelin-port]/api/interpreter/repository/[repository ID]```
    Success code200
    Fail code 500
    diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index c5441ab9baf..2d4728df5a0 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -59,9 +59,6 @@ import org.slf4j.LoggerFactory; import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.google.gson.JsonParseException; -import com.google.gson.JsonParser; import py4j.GatewayServer; @@ -124,12 +121,12 @@ public void open() { // load libraries from Dependency Interpreter URL [] urls = new URL[0]; + List urlList = new LinkedList(); if (depInterpreter != null) { SparkDependencyContext depc = depInterpreter.getDependencyContext(); if (depc != null) { List files = depc.getFiles(); - List urlList = new LinkedList(); if (files != null) { for (File f : files) { try { @@ -138,12 +135,29 @@ public void open() { logger.error("Error", e); } } + } + } + } - urls = urlList.toArray(urls); + String localRepo = getProperty("zeppelin.interpreter.localRepo"); + if (localRepo != null) { + File localRepoDir = new File(localRepo); + if (localRepoDir.exists()) { + File[] files = localRepoDir.listFiles(); + if (files != null) { + for (File f : files) { + try { + urlList.add(f.toURI().toURL()); + } catch (MalformedURLException e) { + logger.error("Error", e); + } + } } } } + urls = urlList.toArray(urls); + ClassLoader oldCl = Thread.currentThread().getContextClassLoader(); try { URLClassLoader newCl = new URLClassLoader(urls, oldCl); diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 7ee6d7ccb60..d8e0f81d1a2 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -438,6 +438,23 @@ public void open() { } } + // add dependency from local repo + String localRepo = getProperty("zeppelin.interpreter.localRepo"); + if (localRepo != null) { + File localRepoDir = new File(localRepo); + if (localRepoDir.exists()) { + File[] files = localRepoDir.listFiles(); + if (files != null) { + for (File f : files) { + if (classpath.length() > 0) { + classpath += File.pathSeparator; + } + classpath += f.getAbsolutePath(); + } + } + } + } + pathSettings.v_$eq(classpath); settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings); @@ -529,7 +546,7 @@ public void open() { throw new InterpreterException(e); } - // add jar + // add jar from DepInterpreter if (depInterpreter != null) { SparkDependencyContext depc = depInterpreter.getDependencyContext(); if (depc != null) { @@ -547,6 +564,25 @@ public void open() { } } } + + // add jar from local repo + if (localRepo != null) { + File localRepoDir = new File(localRepo); + if (localRepoDir.exists()) { + File[] files = localRepoDir.listFiles(); + if (files != null) { + for (File f : files) { + if (f.getName().toLowerCase().endsWith(".jar")) { + sc.addJar(f.getAbsolutePath()); + logger.info("sc.addJar(" + f.getAbsolutePath() + ")"); + } else { + sc.addFile(f.getAbsolutePath()); + logger.info("sc.addFile(" + f.getAbsolutePath() + ")"); + } + } + } + } + } } private List currentClassPath() { diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index 926f3e7546b..02011881b6d 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -33,7 +33,6 @@ import org.apache.spark.SparkContext; import org.apache.spark.sql.SQLContext; -import org.apache.spark.sql.SQLContext.QueryExecution; import org.apache.spark.sql.catalyst.expressions.Attribute; import org.apache.spark.sql.hive.HiveContext; import org.apache.zeppelin.display.AngularObject; @@ -74,127 +73,6 @@ public ZeppelinContext(SparkContext sc, SQLContext sql, public HiveContext hiveContext; private GUI gui; - /** - * Load dependency for interpreter and runtime (driver). - * And distribute them to spark cluster (sc.add()) - * - * @param artifact "group:artifact:version" or file path like "/somepath/your.jar" - * @return - * @throws Exception - */ - public Iterable load(String artifact) throws Exception { - return collectionAsScalaIterable(dep.load(artifact, true)); - } - - /** - * Load dependency and it's transitive dependencies for interpreter and runtime (driver). - * And distribute them to spark cluster (sc.add()) - * - * @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar" - * @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string. - * @return - * @throws Exception - */ - public Iterable load(String artifact, scala.collection.Iterable excludes) - throws Exception { - return collectionAsScalaIterable( - dep.load(artifact, - asJavaCollection(excludes), - true)); - } - - /** - * Load dependency and it's transitive dependencies for interpreter and runtime (driver). - * And distribute them to spark cluster (sc.add()) - * - * @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar" - * @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string. - * @return - * @throws Exception - */ - public Iterable load(String artifact, Collection excludes) throws Exception { - return collectionAsScalaIterable(dep.load(artifact, excludes, true)); - } - - /** - * Load dependency for interpreter and runtime, and then add to sparkContext. - * But not adding them to spark cluster - * - * @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar" - * @return - * @throws Exception - */ - public Iterable loadLocal(String artifact) throws Exception { - return collectionAsScalaIterable(dep.load(artifact, false)); - } - - - /** - * Load dependency and it's transitive dependencies and then add to sparkContext. - * But not adding them to spark cluster - * - * @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar" - * @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string. - * @return - * @throws Exception - */ - public Iterable loadLocal(String artifact, - scala.collection.Iterable excludes) throws Exception { - return collectionAsScalaIterable(dep.load(artifact, - asJavaCollection(excludes), false)); - } - - /** - * Load dependency and it's transitive dependencies and then add to sparkContext. - * But not adding them to spark cluster - * - * @param artifact "groupId:artifactId:version" or file path like "/somepath/your.jar" - * @param excludes exclusion list of transitive dependency. list of "groupId:artifactId" string. - * @return - * @throws Exception - */ - public Iterable loadLocal(String artifact, Collection excludes) - throws Exception { - return collectionAsScalaIterable(dep.load(artifact, excludes, false)); - } - - - /** - * Add maven repository - * - * @param id id of repository ex) oss, local, snapshot - * @param url url of repository. supported protocol : file, http, https - */ - public void addRepo(String id, String url) { - addRepo(id, url, false); - } - - /** - * Add maven repository - * - * @param id id of repository - * @param url url of repository. supported protocol : file, http, https - * @param snapshot true if it is snapshot repository - */ - public void addRepo(String id, String url, boolean snapshot) { - dep.addRepo(id, url, snapshot); - } - - /** - * Remove maven repository by id - * @param id id of repository - */ - public void removeRepo(String id){ - dep.delRepo(id); - } - - /** - * Load dependency only interpreter. - * - * @param name - * @return - */ - public Object input(String name) { return input(name, ""); } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java index 1b20b0fb6f0..7e73c98b77b 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java @@ -42,6 +42,8 @@ import org.sonatype.aether.util.filter.DependencyFilterUtils; import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter; +import scala.Console; + /** * @@ -64,6 +66,8 @@ public SparkDependencyContext(String localRepoPath, String additionalRemoteRepos } public Dependency load(String lib) { + Console.println("DepInterpreter(%dep) deprecated. " + + "Load dependency through GUI interpreter menu instead."); Dependency dep = new Dependency(lib); if (dependencies.contains(dep)) { @@ -74,12 +78,16 @@ public Dependency load(String lib) { } public Repository addRepo(String name) { + Console.println("DepInterpreter(%dep) deprecated. " + + "Add repository through GUI interpreter menu instead."); Repository rep = new Repository(name); repositories.add(rep); return rep; } public void reset() { + Console.println("DepInterpreter(%dep) deprecated. " + + "Remove dependencies and repositories through GUI interpreter menu instead."); dependencies = new LinkedList(); repositories = new LinkedList(); @@ -156,7 +164,7 @@ private List fetchArtifactWithDep(Dependency dep) collectRequest.addRepository(repo); } for (Repository repo : repositories) { - RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl()); + RemoteRepository rr = new RemoteRepository(repo.getId(), "default", repo.getUrl()); rr.setPolicy(repo.isSnapshot(), null); Authentication auth = repo.getAuthentication(); if (auth != null) { diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index 778966f2ba9..7064e7364d5 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -172,16 +172,6 @@ public void testReferencingUndefinedVal() { assertEquals(Code.ERROR, result.code()); } - @Test - public void testZContextDependencyLoading() { - // try to import library does not exist on classpath. it'll fail - assertEquals(InterpreterResult.Code.ERROR, repl.interpret("import org.apache.commons.csv.CSVFormat", context).code()); - - // load library from maven repository and try to import again - repl.interpret("z.load(\"org.apache.commons:commons-csv:1.1\")", context); - assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("import org.apache.commons.csv.CSVFormat", context).code()); - } - @Test public void emptyConfigurationVariablesOnlyForNonSparkProperties() { Properties intpProperty = repl.getProperty(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java index ba8ee16d3a9..f2f3baa14a7 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java @@ -24,7 +24,9 @@ import org.sonatype.aether.RepositorySystem; import org.sonatype.aether.RepositorySystemSession; +import org.sonatype.aether.repository.Authentication; import org.sonatype.aether.repository.RemoteRepository; +import org.sonatype.aether.repository.RepositoryPolicy; import org.sonatype.aether.resolution.ArtifactResult; /** @@ -41,12 +43,32 @@ public AbstractDependencyResolver(String localRepoPath) { repos.add(Booter.newCentralRepository()); // add maven central repos.add(Booter.newLocalRepository()); } + + public List getRepos() { + return this.repos; + } public void addRepo(String id, String url, boolean snapshot) { synchronized (repos) { delRepo(id); RemoteRepository rr = new RemoteRepository(id, "default", url); - rr.setPolicy(snapshot, null); + rr.setPolicy(true, new RepositoryPolicy( + snapshot, + RepositoryPolicy.UPDATE_POLICY_DAILY, + RepositoryPolicy.CHECKSUM_POLICY_WARN)); + repos.add(rr); + } + } + + public void addRepo(String id, String url, boolean snapshot, Authentication auth) { + synchronized (repos) { + delRepo(id); + RemoteRepository rr = new RemoteRepository(id, "default", url); + rr.setPolicy(true, new RepositoryPolicy( + snapshot, + RepositoryPolicy.UPDATE_POLICY_DAILY, + RepositoryPolicy.CHECKSUM_POLICY_WARN)); + rr.setAuthentication(auth); repos.add(rr); } } @@ -54,7 +76,7 @@ public void addRepo(String id, String url, boolean snapshot) { public RemoteRepository delRepo(String id) { synchronized (repos) { Iterator it = repos.iterator(); - if (it.hasNext()) { + while (it.hasNext()) { RemoteRepository repo = it.next(); if (repo.getId().equals(id)) { it.remove(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java index ab4da28fb2b..a3208109ebc 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java @@ -127,7 +127,7 @@ private List fetchArtifactWithDep(Dependency dep) collectRequest.addRepository(mavenCentral); collectRequest.addRepository(mavenLocal); for (Repository repo : repositories) { - RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl()); + RemoteRepository rr = new RemoteRepository(repo.getId(), "default", repo.getUrl()); rr.setPolicy(repo.isSnapshot(), null); collectRequest.addRepository(rr); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index cbe88bc8fda..21e1cbb66e9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -18,6 +18,7 @@ package org.apache.zeppelin.dep; import java.io.File; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; @@ -28,6 +29,7 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonatype.aether.RepositoryException; import org.sonatype.aether.artifact.Artifact; import org.sonatype.aether.collection.CollectRequest; import org.sonatype.aether.graph.Dependency; @@ -56,16 +58,18 @@ public DependencyResolver(String localRepoPath) { super(localRepoPath); } - public List load(String artifact) throws Exception { + public List load(String artifact) + throws RepositoryException, IOException { return load(artifact, new LinkedList()); } - public List load(String artifact, String destPath) throws Exception { + public List load(String artifact, String destPath) + throws RepositoryException, IOException { return load(artifact, new LinkedList(), destPath); } public synchronized List load(String artifact, Collection excludes) - throws Exception { + throws RepositoryException, IOException { if (StringUtils.isBlank(artifact)) { // Should throw here throw new RuntimeException("Invalid artifact to load"); @@ -83,7 +87,7 @@ public synchronized List load(String artifact, Collection excludes } public List load(String artifact, Collection excludes, String destPath) - throws Exception { + throws RepositoryException, IOException { List libs = load(artifact, excludes); // find home dir @@ -105,7 +109,8 @@ public List load(String artifact, Collection excludes, String dest return libs; } - private List loadFromMvn(String artifact, Collection excludes) throws Exception { + private List loadFromMvn(String artifact, Collection excludes) + throws RepositoryException { Collection allExclusions = new LinkedList(); allExclusions.addAll(excludes); allExclusions.addAll(Arrays.asList(exclusions)); @@ -142,7 +147,7 @@ private List loadFromMvn(String artifact, Collection excludes) thr */ @Override public List getArtifactsWithDep(String dependency, - Collection excludes) throws Exception { + Collection excludes) throws RepositoryException { Artifact artifact = new DefaultArtifact(dependency); DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE); PatternExclusionsDependencyFilter exclusionFilter = diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java index 4c2d8670643..d2b00924607 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java @@ -23,13 +23,13 @@ */ public class Repository { private boolean snapshot = false; - private String name; + private String id; private String url; private String username = null; private String password = null; - public Repository(String name){ - this.name = name; + public Repository(String id){ + this.id = id; } public Repository url(String url) { @@ -46,8 +46,8 @@ public boolean isSnapshot() { return snapshot; } - public String getName() { - return name; + public String getId() { + return id; } public String getUrl() { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java index ac251d2a7e3..014b13ea482 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java @@ -127,10 +127,6 @@ public void destroy() { } } - - - - public static Logger logger = LoggerFactory.getLogger(Interpreter.class); private InterpreterGroup interpreterGroup; private URL [] classloaderUrls; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index d8cb22364e2..f1eec08ba1a 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -30,9 +30,7 @@ import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterResult.Type; -import org.apache.zeppelin.interpreter.WrappedInterpreter; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; @@ -53,6 +51,7 @@ public class RemoteInterpreter extends Interpreter { Gson gson = new Gson(); private String interpreterRunner; private String interpreterPath; + private String localRepoPath; private String className; FormType formType; boolean initialized; @@ -61,17 +60,19 @@ public class RemoteInterpreter extends Interpreter { private int maxPoolSize; public RemoteInterpreter(Properties property, - String className, - String interpreterRunner, - String interpreterPath, - int connectTimeout, - int maxPoolSize, - RemoteInterpreterProcessListener remoteInterpreterProcessListener) { + String className, + String interpreterRunner, + String interpreterPath, + String localRepoPath, + int connectTimeout, + int maxPoolSize, + RemoteInterpreterProcessListener remoteInterpreterProcessListener) { super(property); this.className = className; initialized = false; this.interpreterRunner = interpreterRunner; this.interpreterPath = interpreterPath; + this.localRepoPath = localRepoPath; env = new HashMap(); this.connectTimeout = connectTimeout; this.maxPoolSize = maxPoolSize; @@ -79,16 +80,18 @@ public RemoteInterpreter(Properties property, } public RemoteInterpreter(Properties property, - String className, - String interpreterRunner, - String interpreterPath, - Map env, - int connectTimeout, - RemoteInterpreterProcessListener remoteInterpreterProcessListener) { + String className, + String interpreterRunner, + String interpreterPath, + String localRepoPath, + Map env, + int connectTimeout, + RemoteInterpreterProcessListener remoteInterpreterProcessListener) { super(property); this.className = className; this.interpreterRunner = interpreterRunner; this.interpreterPath = interpreterPath; + this.localRepoPath = localRepoPath; this.env = env; this.connectTimeout = connectTimeout; this.maxPoolSize = 10; @@ -110,8 +113,8 @@ public RemoteInterpreterProcess getInterpreterProcess() { if (intpGroup.getRemoteInterpreterProcess() == null) { // create new remote process RemoteInterpreterProcess remoteProcess = new RemoteInterpreterProcess( - interpreterRunner, interpreterPath, env, connectTimeout, - remoteInterpreterProcessListener); + interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, + remoteInterpreterProcessListener); intpGroup.setRemoteInterpreterProcess(remoteProcess); } @@ -143,6 +146,7 @@ private synchronized void init() { try { for (Interpreter intp : this.getInterpreterGroup()) { logger.info("Create remote interpreter {}", intp.getClassName()); + property.put("zeppelin.interpreter.localRepo", localRepoPath); client.createInterpreter(intp.getClassName(), (Map) property); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 5237b0b23bf..2c88894614f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -45,6 +45,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler { private int port = -1; private final String interpreterRunner; private final String interpreterDir; + private final String localRepoDir; private GenericObjectPool clientPool; private Map env; @@ -53,20 +54,28 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler { private int connectTimeout; public RemoteInterpreterProcess(String intpRunner, - String intpDir, - Map env, - int connectTimeout, - RemoteInterpreterProcessListener listener) { - this(intpRunner, intpDir, env, new RemoteInterpreterEventPoller(listener), connectTimeout); + String intpDir, + String localRepoDir, + Map env, + int connectTimeout, + RemoteInterpreterProcessListener listener) { + this(intpRunner, + intpDir, + localRepoDir, + env, + new RemoteInterpreterEventPoller(listener), + connectTimeout); } RemoteInterpreterProcess(String intpRunner, String intpDir, + String localRepoDir, Map env, RemoteInterpreterEventPoller remoteInterpreterEventPoller, int connectTimeout) { this.interpreterRunner = intpRunner; this.interpreterDir = intpDir; + this.localRepoDir = localRepoDir; this.env = env; this.interpreterContextRunnerPool = new InterpreterContextRunnerPool(); referenceCount = new AtomicInteger(0); @@ -89,12 +98,13 @@ public int reference(InterpreterGroup interpreterGroup) { throw new InterpreterException(e1); } - CommandLine cmdLine = CommandLine.parse(interpreterRunner); cmdLine.addArgument("-d", false); cmdLine.addArgument(interpreterDir, false); cmdLine.addArgument("-p", false); cmdLine.addArgument(Integer.toString(port), false); + cmdLine.addArgument("-l", false); + cmdLine.addArgument(localRepoDir, false); executor = new DefaultExecutor(); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java index 33b7e5478a1..a8664ef67fe 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java @@ -17,14 +17,18 @@ package org.apache.zeppelin.dep; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; import java.io.File; +import java.util.Collections; import org.apache.commons.io.FileUtils; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonatype.aether.RepositoryException; public class DependencyResolverTest { private static DependencyResolver resolver; @@ -50,13 +54,47 @@ public static void setUp() throws Exception { public static void tearDown() throws Exception { FileUtils.deleteDirectory(new File(home + "/" + testPath)); FileUtils.deleteDirectory(new File(home + "/" + testCopyPath)); - } - + } + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void testAddRepo() { + int reposCnt = resolver.getRepos().size(); + resolver.addRepo("securecentral", "https://repo1.maven.org/maven2", false); + assertEquals(reposCnt + 1, resolver.getRepos().size()); + } + + @Test + public void testDelRepo() { + int reposCnt = resolver.getRepos().size(); + resolver.delRepo("securecentral"); + resolver.delRepo("badId"); + assertEquals(reposCnt - 1, resolver.getRepos().size()); + } + @Test public void testLoad() throws Exception { - resolver.load("org.apache.commons:commons-lang3:3.4", testCopyPath); + // basic load + resolver.load("com.databricks:spark-csv_2.10:1.3.0", testCopyPath); + assertEquals(new File(home + "/" + testCopyPath).list().length, 4); + FileUtils.cleanDirectory(new File(home + "/" + testCopyPath)); + + // load with exclusions parameter + resolver.load("com.databricks:spark-csv_2.10:1.3.0", + Collections.singletonList("org.scala-lang:scala-library"), testCopyPath); + assertEquals(new File(home + "/" + testCopyPath).list().length, 3); + FileUtils.cleanDirectory(new File(home + "/" + testCopyPath)); + + // load from added repository + resolver.addRepo("sonatype", "https://oss.sonatype.org/content/repositories/agimatec-releases/", false); + resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath); + assertEquals(new File(home + "/" + testCopyPath).list().length, 8); - assertTrue(new File(home + "/" + testPath + "/org/apache/commons/commons-lang3/3.4/").exists()); - assertTrue(new File(home + "/" + testCopyPath + "/commons-lang3-3.4.jar").exists()); + // load invalid artifact + resolver.delRepo("sonatype"); + exception.expect(RepositoryException.class); + resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath); } } \ No newline at end of file diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index 4cd974dbb3c..385b9d6f36b 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -64,14 +64,15 @@ public void setUp() throws Exception { Properties p = new Properties(); intp = new RemoteInterpreter( - p, - MockInterpreterAngular.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), - "fake", - env, - 10 * 1000, - null - ); + p, + MockInterpreterAngular.class.getName(), + new File("../bin/interpreter.sh").getAbsolutePath(), + "fake", + "fakeRepo", + env, + 10 * 1000, + null + ); intpGroup.add(intp); intp.setInterpreterGroup(intpGroup); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java index 623a0379496..f229f6bbfc4 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java @@ -55,14 +55,14 @@ public void tearDown() throws Exception { private RemoteInterpreter createMockInterpreter() { RemoteInterpreter intp = new RemoteInterpreter( - new Properties(), - MockInterpreterOutputStream.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), - "fake", - env, - 10 * 1000, - this); - + new Properties(), + MockInterpreterOutputStream.class.getName(), + new File("../bin/interpreter.sh").getAbsolutePath(), + "fake", + "fakeRepo", + env, + 10 * 1000, + this); intpGroup.add(intp); intp.setInterpreterGroup(intpGroup); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index abee5b80207..7beaee111f6 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -33,7 +33,7 @@ public class RemoteInterpreterProcessTest { public void testStartStop() { InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess( - "../bin/interpreter.sh", "nonexists", new HashMap(), + "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap(), 10 * 1000, null); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); @@ -50,7 +50,7 @@ public void testStartStop() { public void testClientFactory() throws Exception { InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess( - "../bin/interpreter.sh", "nonexists", new HashMap(), + "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap(), mock(RemoteInterpreterEventPoller.class), 10 * 1000); rip.reference(intpGroup); assertEquals(0, rip.getNumActiveClient()); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index 034a676c2ed..82ca8d4bb0a 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -69,6 +69,7 @@ private RemoteInterpreter createMockInterpreterA(Properties p) { MockInterpreterA.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", + "fakeRepo", env, 10 * 1000, null); @@ -80,6 +81,7 @@ private RemoteInterpreter createMockInterpreterB(Properties p) { MockInterpreterB.class.getName(), new File("../bin/interpreter.sh").getAbsolutePath(), "fake", + "fakeRepo", env, 10 * 1000, null); @@ -164,27 +166,27 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException Properties p = new Properties(); RemoteInterpreter intpA = new RemoteInterpreter( - p, - MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), - "fake", - env, - 10 * 1000, - null - ); + p, + MockInterpreterA.class.getName(), + new File("../bin/interpreter.sh").getAbsolutePath(), + "fake", + "fakeRepo", + env, + 10 * 1000, + null); intpGroup.add(intpA); intpA.setInterpreterGroup(intpGroup); RemoteInterpreter intpB = new RemoteInterpreter( - p, - MockInterpreterB.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), - "fake", - env, - 10 * 1000, - null - ); + p, + MockInterpreterB.class.getName(), + new File("../bin/interpreter.sh").getAbsolutePath(), + "fake", + "fakeRepo", + env, + 10 * 1000, + null); intpGroup.add(intpB); intpB.setInterpreterGroup(intpGroup); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index 05bc6763363..d46b8cf162e 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -64,14 +64,14 @@ public void test() throws Exception { env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); final RemoteInterpreter intpA = new RemoteInterpreter( - p, - MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), - "fake", - env, - 10 * 1000, - null - ); + p, + MockInterpreterA.class.getName(), + new File("../bin/interpreter.sh").getAbsolutePath(), + "fake", + "fakeRepo", + env, + 10 * 1000, + null); intpGroup.add(intpA); intpA.setInterpreterGroup(intpGroup); @@ -148,14 +148,14 @@ public void testAbortOnPending() throws Exception { env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); final RemoteInterpreter intpA = new RemoteInterpreter( - p, - MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), - "fake", - env, - 10 * 1000, - null - ); + p, + MockInterpreterA.class.getName(), + new File("../bin/interpreter.sh").getAbsolutePath(), + "fake", + "fakeRepo", + env, + 10 * 1000, + null); intpGroup.add(intpA); intpA.setInterpreterGroup(intpGroup); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index e1691adb4a9..50d7d643c9f 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -33,6 +33,7 @@ import javax.ws.rs.core.Response.Status; import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.zeppelin.dep.Repository; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; import org.apache.zeppelin.rest.message.NewInterpreterSettingRequest; @@ -42,6 +43,8 @@ import org.slf4j.LoggerFactory; import com.google.gson.Gson; +import org.sonatype.aether.RepositoryException; +import org.sonatype.aether.repository.RemoteRepository; /** * Interpreter Rest API @@ -85,17 +88,34 @@ public Response listSettings() { */ @POST @Path("setting") - public Response newSettings(String message) throws InterpreterException, IOException { - NewInterpreterSettingRequest request = gson.fromJson(message, - NewInterpreterSettingRequest.class); - Properties p = new Properties(); - p.putAll(request.getProperties()); - // Option is deprecated from API, always use remote = true - InterpreterGroup interpreterGroup = interpreterFactory.add(request.getName(), - request.getGroup(), new InterpreterOption(true), p); - InterpreterSetting setting = interpreterFactory.get(interpreterGroup.getId()); - logger.info("new setting created with " + setting.id()); - return new JsonResponse(Status.CREATED, "", setting ).build(); + public Response newSettings(String message) { + try { + NewInterpreterSettingRequest request = gson.fromJson(message, + NewInterpreterSettingRequest.class); + Properties p = new Properties(); + p.putAll(request.getProperties()); + // Option is deprecated from API, always use remote = true + InterpreterGroup interpreterGroup = interpreterFactory.add(request.getName(), + request.getGroup(), + request.getDependencies(), + new InterpreterOption(true), + p); + InterpreterSetting setting = interpreterFactory.get(interpreterGroup.getId()); + logger.info("new setting created with {}", setting.id()); + return new JsonResponse(Status.CREATED, "", setting).build(); + } catch (InterpreterException e) { + logger.error("Exception in InterpreterRestApi while creating ", e); + return new JsonResponse( + Status.NOT_FOUND, + e.getMessage(), + ExceptionUtils.getStackTrace(e)).build(); + } catch (IOException | RepositoryException e) { + logger.error("Exception in InterpreterRestApi while creating ", e); + return new JsonResponse( + Status.INTERNAL_SERVER_ERROR, + e.getMessage(), + ExceptionUtils.getStackTrace(e)).build(); + } } @PUT @@ -104,16 +124,18 @@ public Response updateSetting(String message, @PathParam("settingId") String set logger.info("Update interpreterSetting {}", settingId); try { - UpdateInterpreterSettingRequest p = gson.fromJson(message, + UpdateInterpreterSettingRequest request = gson.fromJson(message, UpdateInterpreterSettingRequest.class); // Option is deprecated from API, always use remote = true interpreterFactory.setPropertyAndRestart(settingId, - new InterpreterOption(true), p.getProperties()); + new InterpreterOption(true), + request.getProperties(), + request.getDependencies()); } catch (InterpreterException e) { logger.error("Exception in InterpreterRestApi while updateSetting ", e); return new JsonResponse( Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); - } catch (IOException e) { + } catch (IOException | RepositoryException e) { logger.error("Exception in InterpreterRestApi while updateSetting ", e); return new JsonResponse( Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); @@ -165,4 +187,59 @@ public Response listInterpreter(String message) { Map m = Interpreter.registeredInterpreters; return new JsonResponse(Status.OK, "", m).build(); } + + /** + * List of dependency resolving repositories + * @return + */ + @GET + @Path("repository") + public Response listRepositories() { + List interpreterRepositories = null; + interpreterRepositories = interpreterFactory.getRepositories(); + return new JsonResponse(Status.OK, "", interpreterRepositories).build(); + } + + /** + * Add new repository + * @param message + * @return + */ + @POST + @Path("repository") + public Response addRepository(String message) { + try { + Repository request = gson.fromJson(message, Repository.class); + interpreterFactory.addRepository( + request.getId(), + request.getUrl(), + request.isSnapshot(), + request.getAuthentication()); + logger.info("New repository {} added", request.getId()); + } catch (Exception e) { + logger.error("Exception in InterpreterRestApi while adding repository ", e); + return new JsonResponse( + Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); + } + return new JsonResponse(Status.CREATED).build(); + } + + /** + * Delete repository + * @param repoId + * @return + */ + @DELETE + @Path("repository/{repoId}") + public Response removeRepository(@PathParam("repoId") String repoId) { + logger.info("Remove repository {}", repoId); + try { + interpreterFactory.removeRepository(repoId); + } catch (Exception e) { + logger.error("Exception in InterpreterRestApi while removing repository ", e); + return new JsonResponse( + Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); + } + return new JsonResponse(Status.OK).build(); + } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java index 36f80f68b19..22eb25f98af 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java @@ -17,9 +17,10 @@ package org.apache.zeppelin.rest.message; +import java.util.List; import java.util.Map; -import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.dep.Dependency; /** * NewInterpreterSetting rest api request message @@ -30,6 +31,7 @@ public class NewInterpreterSettingRequest { String group; // option was deprecated Map properties; + List dependencies; public NewInterpreterSettingRequest() { @@ -47,4 +49,7 @@ public Map getProperties() { return properties; } + public List getDependencies() { + return dependencies; + } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateInterpreterSettingRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateInterpreterSettingRequest.java index f1f496ae8d6..a3f71ea9ad5 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateInterpreterSettingRequest.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/UpdateInterpreterSettingRequest.java @@ -17,27 +17,30 @@ package org.apache.zeppelin.rest.message; +import java.util.List; import java.util.Properties; -import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.dep.Dependency; /** - * + * UpdateInterpreterSetting rest api request message */ public class UpdateInterpreterSettingRequest { - // option was deprecated Properties properties; + List dependencies; - public UpdateInterpreterSettingRequest(InterpreterOption option, - Properties properties) { - super(); + public UpdateInterpreterSettingRequest(Properties properties, + List dependencies) { this.properties = properties; + this.dependencies = dependencies; } public Properties getProperties() { return properties; } - + public List getDependencies() { + return dependencies; + } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 760298fe34a..40e4d14d1df 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -76,7 +76,8 @@ public class ZeppelinServer extends Application { public ZeppelinServer() throws Exception { ZeppelinConfiguration conf = ZeppelinConfiguration.create(); - this.depResolver = new DependencyResolver(conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO)); + this.depResolver = new DependencyResolver( + conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO)); this.schedulerFactory = new SchedulerFactory(); this.replFactory = new InterpreterFactory(conf, notebookWsServer, notebookWsServer, depResolver); diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java index 44769406e2a..c3386dd455d 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java @@ -17,20 +17,17 @@ package org.apache.zeppelin; -import com.google.common.base.Function; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.openqa.selenium.*; -import org.openqa.selenium.support.ui.FluentWait; -import org.openqa.selenium.support.ui.Wait; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Test Zeppelin with web browser. @@ -202,4 +199,49 @@ public void testAngularDisplay() throws InterruptedException{ } } + + @Test + public void testSparkInterpreterDependencyLoading() { + // navigate to interpreter page + WebElement interpreterLink = driver.findElement(By.linkText("Interpreter")); + interpreterLink.click(); + + // add new dependency to spark interpreter + WebElement sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"), + MAX_BROWSER_TIMEOUT_SEC); + sparkEditBtn.click(); + WebElement depArtifact = driver.findElement(By.xpath("//input[@ng-model='setting.depArtifact']")); + String artifact = "org.apache.commons:commons-csv:1.1"; + depArtifact.sendKeys(artifact); + driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit(); + driver.switchTo().alert().accept(); + + driver.navigate().back(); + createNewNote(); + + // wait for first paragraph's " READY " status text + waitForParagraph(1, "READY"); + + WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea")); + + paragraph1Editor.sendKeys("import org.apache.commons.csv.CSVFormat"); + paragraph1Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER)); + waitForParagraph(1, "FINISHED"); + + // check expected text + assertTrue(waitForText("import org.apache.commons.csv.CSVFormat", + By.xpath(getParagraphXPath(1) + "//div[starts-with(@id, 'p') and contains(@id, 'text')]/div"))); + + // reset dependency + interpreterLink.click(); + sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"), + MAX_BROWSER_TIMEOUT_SEC); + sparkEditBtn.click(); + WebElement testDepRemoveBtn = driver.findElement(By.xpath("//tr[descendant::text()[contains(.,'" + + artifact + "')]]/td[3]/div")); + sleep(5000, true); + testDepRemoveBtn.click(); + driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit(); + driver.switchTo().alert().accept(); + } } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 0c4199debf6..81f161af4f6 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -22,6 +22,7 @@ import java.lang.ref.WeakReference; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.LinkedList; import java.util.List; import java.util.Properties; import java.util.concurrent.ExecutorService; @@ -35,6 +36,7 @@ import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.httpclient.methods.PutMethod; import org.apache.commons.httpclient.methods.RequestEntity; +import org.apache.zeppelin.dep.Dependency; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterSetting; @@ -48,6 +50,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParseException; import com.google.gson.JsonParser; +import org.sonatype.aether.RepositoryException; public abstract class AbstractTestRestApi { @@ -363,10 +366,14 @@ protected void describeMismatchSafely(String item, Description description) { } //Create new Setting and return Setting ID - protected String createTempSetting(String tempName) throws IOException { - - InterpreterGroup interpreterGroup = ZeppelinServer.notebook.getInterpreterFactory().add(tempName,"newGroup", - new InterpreterOption(false),new Properties()); + protected String createTempSetting(String tempName) + throws IOException, RepositoryException { + InterpreterGroup interpreterGroup = ZeppelinServer.notebook.getInterpreterFactory() + .add(tempName, + "newGroup", + new LinkedList(), + new InterpreterOption(false), + new Properties()); return interpreterGroup.getId(); } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java new file mode 100644 index 00000000000..8886add53f6 --- /dev/null +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.rest; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.commons.httpclient.methods.DeleteMethod; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.methods.PostMethod; +import org.apache.commons.httpclient.methods.PutMethod; +import org.apache.commons.lang3.StringUtils; +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.server.ZeppelinServer; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import static org.junit.Assert.*; + +/** + * Zeppelin interpreter rest api tests + */ +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class InterpreterRestApiTest extends AbstractTestRestApi { + Gson gson = new Gson(); + + @BeforeClass + public static void init() throws Exception { + AbstractTestRestApi.startUp(); + } + + @AfterClass + public static void destroy() throws Exception { + AbstractTestRestApi.shutDown(); + } + + @Test + public void getAvailableInterpreters() throws IOException { + // when + GetMethod get = httpGet("/interpreter"); + + // then + assertThat(get, isAllowed()); + Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>() { + }.getType()); + Map body = (Map) resp.get("body"); + assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getRegisteredInterpreterList().size(), body.size()); + get.releaseConnection(); + } + + @Test + public void getSettings() throws IOException { + // when + GetMethod get = httpGet("/interpreter/setting"); + + // then + Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>() { + }.getType()); + assertThat(get, isAllowed()); + get.releaseConnection(); + } + + @Test + public void testSettingsCRUD() throws IOException { + // Call Create Setting REST API + String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," + + "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," + + "\"dependencies\":[]}"; + PostMethod post = httpPost("/interpreter/setting/", jsonRequest); + LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString()); + assertThat("test create method:", post, isCreated()); + + Map resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken>() { + }.getType()); + Map body = (Map) resp.get("body"); + //extract id from body string {id=2AWMQDNX7, name=md2, group=md, + String newSettingId = body.toString().split(",")[0].split("=")[1]; + post.releaseConnection(); + + // Call Update Setting REST API + jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"}," + + "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," + + "\"dependencies\":[]}"; + PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest); + LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString()); + assertThat("test update method:", put, isAllowed()); + put.releaseConnection(); + + // Call Delete Setting REST API + DeleteMethod delete = httpDelete("/interpreter/setting/" + newSettingId); + LOG.info("testSettingCRUD delete response\n" + delete.getResponseBodyAsString()); + assertThat("Test delete method:", delete, isAllowed()); + delete.releaseConnection(); + } + + @Test + public void testInterpreterAutoBinding() throws IOException { + // create note + Note note = ZeppelinServer.notebook.createNote(); + + // check interpreter is binded + GetMethod get = httpGet("/notebook/interpreter/bind/" + note.id()); + assertThat(get, isAllowed()); + get.addRequestHeader("Origin", "http://localhost"); + Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>() { + }.getType()); + List> body = (List>) resp.get("body"); + assertTrue(0 < body.size()); + + get.releaseConnection(); + //cleanup + ZeppelinServer.notebook.removeNote(note.getId()); + } + + @Test + public void testInterpreterRestart() throws IOException, InterruptedException { + // create new note + Note note = ZeppelinServer.notebook.createNote(); + note.addParagraph(); + Paragraph p = note.getLastParagraph(); + Map config = p.getConfig(); + config.put("enabled", true); + + // run markdown paragraph + p.setConfig(config); + p.setText("%md markdown"); + note.run(p.getId()); + while (p.getStatus() != Status.FINISHED) { + Thread.sleep(100); + } + assertEquals("

    markdown

    \n", p.getResult().message()); + + + // restart interpreter + for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) { + if (setting.getName().equals("md")) { + // Call Restart Interpreter REST API + PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), ""); + assertThat("test interpreter restart:", put, isAllowed()); + put.releaseConnection(); + break; + } + } + + // run markdown paragraph, again + p = note.addParagraph(); + p.setConfig(config); + p.setText("%md markdown restarted"); + note.run(p.getId()); + while (p.getStatus() != Status.FINISHED) { + Thread.sleep(100); + } + assertEquals("

    markdown restarted

    \n", p.getResult().message()); + //cleanup + ZeppelinServer.notebook.removeNote(note.getId()); + } + + @Test + public void testListRepository() throws IOException { + GetMethod get = httpGet("/interpreter/repository"); + assertThat(get, isAllowed()); + get.releaseConnection(); + } + + @Test + public void testAddDeleteRepository() throws IOException { + // Call create repository REST API + String repoId = "securecentral"; + String jsonRequest = "{\"id\":\"" + repoId + + "\",\"url\":\"https://repo1.maven.org/maven2\",\"snapshot\":\"false\"}"; + + PostMethod post = httpPost("/interpreter/repository/", jsonRequest); + assertThat("Test create method:", post, isCreated()); + post.releaseConnection(); + + // Call delete repository REST API + DeleteMethod delete = httpDelete("/interpreter/repository/" + repoId); + assertThat("Test delete method:", delete, isAllowed()); + delete.releaseConnection(); + } +} diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index d8049cc6b80..3c7c7d0e67e 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -72,124 +72,6 @@ public void getApiRoot() throws IOException { httpGetRoot.releaseConnection(); } - - @Test - public void getAvailableInterpreters() throws IOException { - // when - GetMethod get = httpGet("/interpreter"); - - // then - assertThat(get, isAllowed()); - Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>() { - }.getType()); - Map body = (Map) resp.get("body"); - assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getRegisteredInterpreterList().size(), body.size()); - get.releaseConnection(); - } - - @Test - public void getSettings() throws IOException { - // when - GetMethod get = httpGet("/interpreter/setting"); - - // then - Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>() { - }.getType()); - assertThat(get, isAllowed()); - get.releaseConnection(); - } - - @Test - public void testSettingsCRUD() throws IOException { - // Call Create Setting REST API - String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"},\"" + - "interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]}"; - PostMethod post = httpPost("/interpreter/setting/", jsonRequest); - LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString()); - assertThat("test create method:", post, isCreated()); - - Map resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken>() { - }.getType()); - Map body = (Map) resp.get("body"); - //extract id from body string {id=2AWMQDNX7, name=md2, group=md, - String newSettingId = body.toString().split(",")[0].split("=")[1]; - post.releaseConnection(); - - // Call Update Setting REST API - jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"},\"" + - "interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]}"; - PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest); - LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString()); - assertThat("test update method:", put, isAllowed()); - put.releaseConnection(); - - // Call Delete Setting REST API - DeleteMethod delete = httpDelete("/interpreter/setting/" + newSettingId); - LOG.info("testSettingCRUD delete response\n" + delete.getResponseBodyAsString()); - assertThat("Test delete method:", delete, isAllowed()); - delete.releaseConnection(); - } - @Test - public void testInterpreterAutoBinding() throws IOException { - // create note - Note note = ZeppelinServer.notebook.createNote(); - - // check interpreter is binded - GetMethod get = httpGet("/notebook/interpreter/bind/"+note.id()); - assertThat(get, isAllowed()); - get.addRequestHeader("Origin", "http://localhost"); - Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>(){}.getType()); - List> body = (List>) resp.get("body"); - assertTrue(0 < body.size()); - - get.releaseConnection(); - //cleanup - ZeppelinServer.notebook.removeNote(note.getId()); - } - - @Test - public void testInterpreterRestart() throws IOException, InterruptedException { - // create new note - Note note = ZeppelinServer.notebook.createNote(); - note.addParagraph(); - Paragraph p = note.getLastParagraph(); - Map config = p.getConfig(); - config.put("enabled", true); - - // run markdown paragraph - p.setConfig(config); - p.setText("%md markdown"); - note.run(p.getId()); - while (p.getStatus() != Status.FINISHED) { - Thread.sleep(100); - } - assertEquals("

    markdown

    \n", p.getResult().message()); - - - // restart interpreter - for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) { - if (setting.getName().equals("md")) { - // Call Restart Interpreter REST API - PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), ""); - assertThat("test interpreter restart:", put, isAllowed()); - put.releaseConnection(); - break; - } - } - - // run markdown paragraph, again - p = note.addParagraph(); - p.setConfig(config); - p.setText("%md markdown restarted"); - note.run(p.getId()); - while (p.getStatus() != Status.FINISHED) { - Thread.sleep(100); - } - assertEquals("

    markdown restarted

    \n", p.getResult().message()); - //cleanup - ZeppelinServer.notebook.removeNote(note.getId()); - } - @Test public void testGetNotebookInfo() throws IOException { LOG.info("testGetNotebookInfo"); @@ -303,7 +185,7 @@ private void testNotebookCreate(String noteName) throws IOException { } @Test - public void testDeleteNote() throws IOException { + public void testDeleteNote() throws IOException { LOG.info("testDeleteNote"); //Create note and get ID Note note = ZeppelinServer.notebook.createNote(); diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index d13faf0b285..33a9daf8527 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -81,7 +81,8 @@ ngToastProvider.configure({ dismissButton: true, dismissOnClick: false, - timeout: 6000 + timeout: 6000, + verticalPosition: 'bottom' }); }); diff --git a/zeppelin-web/src/app/home/home.css b/zeppelin-web/src/app/home/home.css index 02021424415..dada8982d50 100644 --- a/zeppelin-web/src/app/home/home.css +++ b/zeppelin-web/src/app/home/home.css @@ -275,31 +275,6 @@ kbd { border-radius: 2px; } -/* - ngToast Style -*/ - -.ng-toast .alert { - color: white !important; - border: none !important; -} - -.ng-toast .alert-danger { - background: #A94442 !important; -} - -.ng-toast .alert-warning { - background: #CE9532 !important; -} - -.ng-toast .alert-info { - background: #589EC1 !important; -} - -.ng-toast .alert-success { - background: #428443 !important; -} - /* temporary fix for bootstrap issue (https://github.com/twbs/bootstrap/issues/5865) This part should be removed when new version of bootstrap handles this issue. diff --git a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html index 38ae7ccaee9..6f46c4e0e66 100644 --- a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html +++ b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html @@ -12,10 +12,10 @@ limitations under the License. -->
    -
    -
    -
    -
    +
    +
    +
    +

    Create new interpreter

    @@ -67,6 +67,50 @@

    Create new interpreter

    + Dependencies + + + + + + + + + + + + + + + + + + +
    artifactexcludeaction
    + + + + +
    +
    +
    + + + + +
    +
    +
    + Save diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index 396a552ef47..1e3800aa49f 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -15,37 +15,42 @@ 'use strict'; angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, $route, $routeParams, $location, $rootScope, - $http, baseUrlSrv) { + $http, baseUrlSrv, ngToast) { var interpreterSettingsTmp = []; $scope.interpreterSettings = []; $scope.availableInterpreters = {}; $scope.showAddNewSetting = false; + $scope.showRepositoryInfo = false; $scope._ = _; var getInterpreterSettings = function() { $http.get(baseUrlSrv.getRestApiBase()+'/interpreter/setting'). - success(function(data, status, headers, config) { - $scope.interpreterSettings = data.body; - }). - error(function(data, status, headers, config) { - console.log('Error %o %o', status, data.message); - }); + success(function(data, status, headers, config) { + $scope.interpreterSettings = data.body; + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); }; var getAvailableInterpreters = function() { $http.get(baseUrlSrv.getRestApiBase()+'/interpreter'). - success(function(data, status, headers, config) { - $scope.availableInterpreters = data.body; - }). - error(function(data, status, headers, config) { - console.log('Error %o %o', status, data.message); - }); + success(function(data, status, headers, config) { + $scope.availableInterpreters = data.body; + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); }; var emptyNewProperty = function(object) { angular.extend(object, {propertyValue: '', propertyKey: ''}); }; + var emptyNewDependency = function(object) { + angular.extend(object, {depArtifact: '', depExclude: ''}); + }; + var removeTMPSettings = function(index) { interpreterSettingsTmp.splice(index, 1); }; @@ -55,29 +60,34 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, interpreterSettingsTmp[index] = angular.copy($scope.interpreterSettings[index]); }; - $scope.updateInterpreterSetting = function(settingId) { - BootstrapDialog.confirm({ - closable: true, - title: '', - message: 'Do you want to update this interpreter and restart with new settings?', - callback: function(result) { - if (result) { - var index = _.findIndex($scope.interpreterSettings, {'id': settingId}); - var request = { - properties: angular.copy($scope.interpreterSettings[index].properties), - }; - - $http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/' + settingId, request). - success(function (data, status, headers, config) { - $scope.interpreterSettings[index] = data.body; - removeTMPSettings(index); - }). - error(function (data, status, headers, config) { - console.log('Error %o %o', status, data.message); - }); - } + $scope.updateInterpreterSetting = function(form, settingId) { + var result = confirm('Do you want to update this interpreter and restart with new settings?'); + if (result) { + var index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + var setting = $scope.interpreterSettings[index]; + if (setting.propertyKey !== '' || setting.propertyKey) { + $scope.addNewInterpreterProperty(settingId); } - }); + if (setting.depArtifact !== '' || setting.depArtifact) { + $scope.addNewInterpreterDependency(settingId); + } + + var request = { + properties: angular.copy(setting.properties), + dependencies: angular.copy(setting.dependencies) + }; + + $http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/' + settingId, request). + success(function (data, status, headers, config) { + $scope.interpreterSettings[index] = data.body; + removeTMPSettings(index); + }). + error(function (data, status, headers, config) { + console.log('Error %o %o', status, data.message); + ngToast.danger(data.message); + form.$show(); + }); + } }; $scope.resetInterpreterSetting = function(settingId){ @@ -117,8 +127,8 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, var intpInfo = el[i]; for (var key in intpInfo) { properties[key] = { - value : intpInfo[key].defaultValue, - description : intpInfo[key].description + value: intpInfo[key].defaultValue, + description: intpInfo[key].description }; } } @@ -165,32 +175,46 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, return; } - var newSetting = angular.copy($scope.newInterpreterSetting); + var newSetting = $scope.newInterpreterSetting; + if (newSetting.propertyKey !== '' || newSetting.propertyKey) { + $scope.addNewInterpreterProperty(); + } + if (newSetting.depArtifact !== '' || newSetting.depArtifact) { + $scope.addNewInterpreterDependency(); + } + + var request = angular.copy($scope.newInterpreterSetting); - for (var p in $scope.newInterpreterSetting.properties) { - newSetting.properties[p] = $scope.newInterpreterSetting.properties[p].value; + // Change properties to proper request format + var newProperties = {}; + for (var p in newSetting.properties) { + newProperties[p] = newSetting.properties[p].value; } + request.properties = newProperties; - $http.post(baseUrlSrv.getRestApiBase()+'/interpreter/setting', newSetting). - success(function(data, status, headers, config) { - $scope.resetNewInterpreterSetting(); - getInterpreterSettings(); - $scope.showAddNewSetting = false; - }). - error(function(data, status, headers, config) { - console.log('Error %o %o', status, data.message); - }); + $http.post(baseUrlSrv.getRestApiBase() + '/interpreter/setting', request). + success(function(data, status, headers, config) { + $scope.resetNewInterpreterSetting(); + getInterpreterSettings(); + $scope.showAddNewSetting = false; + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + ngToast.danger(data.message); + }); }; $scope.cancelInterpreterSetting = function() { $scope.showAddNewSetting = false; + $scope.resetNewInterpreterSetting(); }; $scope.resetNewInterpreterSetting = function() { $scope.newInterpreterSetting = { - name : undefined, - group : undefined, - properties : {} + name: undefined, + group: undefined, + properties: {}, + dependencies: [] }; emptyNewProperty($scope.newInterpreterSetting); }; @@ -205,6 +229,21 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, } }; + $scope.removeInterpreterDependency = function(artifact, settingId) { + if (settingId === undefined) { + $scope.newInterpreterSetting.dependencies = _.reject($scope.newInterpreterSetting.dependencies, + function(el) { + return el.groupArtifactVersion === artifact; + }); + } else { + var index = _.findIndex($scope.interpreterSettings, {'id': settingId}); + $scope.interpreterSettings[index].dependencies = _.reject($scope.interpreterSettings[index].dependencies, + function(el) { + return el.groupArtifactVersion === artifact; + }); + } + }; + $scope.addNewInterpreterProperty = function(settingId) { if(settingId === undefined) { // Add new property from create form @@ -230,10 +269,126 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', function($scope, } }; + $scope.addNewInterpreterDependency = function(settingId) { + if(settingId === undefined) { + // Add new dependency from create form + if (!$scope.newInterpreterSetting.depArtifact || $scope.newInterpreterSetting.depArtifact === '') { + return; + } + + // overwrite if artifact already exists + var newSetting = $scope.newInterpreterSetting; + for(var d in newSetting.dependencies) { + if (newSetting.dependencies[d].groupArtifactVersion === newSetting.depArtifact) { + newSetting.dependencies[d] = { + 'groupArtifactVersion': newSetting.depArtifact, + 'exclusions': newSetting.depExclude + }; + newSetting.dependencies.splice(d, 1); + } + } + + newSetting.dependencies.push({ + 'groupArtifactVersion': newSetting.depArtifact, + 'exclusions': (newSetting.depExclude === '')? []: newSetting.depExclude + }); + emptyNewDependency(newSetting); + } + else { + // Add new dependency from edit form + var index = _.findIndex($scope.interpreterSettings, { 'id': settingId }); + var setting = $scope.interpreterSettings[index]; + if (!setting.depArtifact || setting.depArtifact === '') { + return; + } + + // overwrite if artifact already exists + for(var dep in setting.dependencies) { + if (setting.dependencies[dep].groupArtifactVersion === setting.depArtifact) { + setting.dependencies[dep] = { + 'groupArtifactVersion': setting.depArtifact, + 'exclusions': setting.depExclude + }; + setting.dependencies.splice(dep, 1); + } + } + + setting.dependencies.push({ + 'groupArtifactVersion': setting.depArtifact, + 'exclusions': (setting.depExclude === '')? []: setting.depExclude + }); + emptyNewDependency(setting); + } + }; + + $scope.resetNewRepositorySetting = function() { + $scope.newRepoSetting = { + id: undefined, + url: undefined, + snapshot: false, + username: undefined, + password: undefined + }; + }; + + var getRepositories = function() { + $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/repository'). + success(function(data, status, headers, config) { + $scope.repositories = data.body; + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); + }; + + $scope.addNewRepository = function() { + var request = angular.copy($scope.newRepoSetting); + + $http.post(baseUrlSrv.getRestApiBase() + '/interpreter/repository', request). + success(function(data, status, headers, config) { + getRepositories(); + $scope.resetNewRepositorySetting(); + angular.element('#repoModal').modal('hide'); + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', headers, config); + }); + }; + + $scope.removeRepository = function(repoId) { + BootstrapDialog.confirm({ + closable: true, + title: '', + message: 'Do you want to delete this repository?', + callback: function(result) { + if (result) { + $http.delete(baseUrlSrv.getRestApiBase()+'/interpreter/repository/' + repoId). + success(function(data, status, headers, config) { + var index = _.findIndex($scope.repositories, { 'id': repoId }); + $scope.repositories.splice(index, 1); + }). + error(function(data, status, headers, config) { + console.log('Error %o %o', status, data.message); + }); + } + } + }); + }; + + $scope.isDefaultRepository = function(repoId) { + if (repoId === 'central' || repoId === 'local') { + return true; + } else { + return false; + } + }; + var init = function() { $scope.resetNewInterpreterSetting(); + $scope.resetNewRepositorySetting(); getInterpreterSettings(); getAvailableInterpreters(); + getRepositories(); }; init(); diff --git a/zeppelin-web/src/app/interpreter/interpreter.css b/zeppelin-web/src/app/interpreter/interpreter.css index b6fe133fcf5..8695f949a05 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.css +++ b/zeppelin-web/src/app/interpreter/interpreter.css @@ -34,6 +34,13 @@ font-size: 12px; } +.interpreter input { + width: 100%; + display: block; + height: 23px; + border: 1px solid #CCCCCC; +} + .interpreter .interpreter-title { font-size: 20px; font-weight: bold; @@ -81,6 +88,11 @@ float: left; } -.empty-properties-message { +.gray40-message { color: #666; } + +.blackOpc:hover { + color: #000; + opacity: .5; +} diff --git a/zeppelin-web/src/app/interpreter/interpreter.html b/zeppelin-web/src/app/interpreter/interpreter.html index 57984dab41d..f1f9a02696a 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.html +++ b/zeppelin-web/src/app/interpreter/interpreter.html @@ -18,11 +18,18 @@

    Interpreters

    - - Create - +
    + + Create + + + + +
    @@ -31,6 +38,42 @@

    + +
    +
    +
    +

    Repositories

    +

    Available repository lists. These repositories are used to resolve external dependencies of interpreter.

    + +
    +
    +
    @@ -66,8 +109,8 @@

    {{setting.name}}

    -
    - Currently there are no properties set for this interpreter +
    + Currently there are no properties and dependencies set for this interpreter
    Properties
    @@ -96,8 +139,7 @@
    Properties
    - + pu-elastic-input-minwidth="180px" /> @@ -109,9 +151,66 @@
    Properties
    -
    -
    + +
    +
    Dependencies
    +

    + These dependencies will be added to classpath when interpreter process starts.

    + + + + + + + + + + + + + + + + + + +
    artifactexcludeaction
    + + {{dep.groupArtifactVersion | breakFilter}} + + + +
    {{dep.exclusions.join()}}
    +     		+
    +
    +
    + + + + +
    +
    +
    + + +

    Add New Repository

    +
    + + +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    +
    + + +
    + +
    +
    +
    diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index a8e082c80a6..1f1166b1bd5 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -63,8 +63,8 @@ +

    You are using an outdated browser. Please upgrade your browser to improve your experience.

    +
    @@ -81,9 +81,9 @@ + + + diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 9e606ee2105..421b3d4961d 100755 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -346,6 +346,10 @@ public String getInterpreterRemoteRunnerPath() { return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER); } + public String getInterpreterLocalRepoPath() { + return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO); + } + public String getRelativeDir(ConfVars c) { return getRelativeDir(getString(c)); } @@ -456,6 +460,7 @@ public static enum ConfVars { + "org.apache.zeppelin.scalding.ScaldingInterpreter," + "org.apache.zeppelin.jdbc.JDBCInterpreter"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), + ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"), ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000), ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10), ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"), diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 3cd125799f0..3761709c49d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -19,10 +19,13 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; + +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.NullArgumentException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.Dependency; import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; @@ -34,6 +37,9 @@ import org.apache.zeppelin.scheduler.Job.Status; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonatype.aether.RepositoryException; +import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.RemoteRepository; import java.io.*; import java.lang.reflect.Constructor; @@ -60,6 +66,7 @@ public class InterpreterFactory { new HashMap(); private Map> interpreterBindings = new HashMap>(); + private List interpreterRepositories; private Gson gson; @@ -68,13 +75,13 @@ public class InterpreterFactory { AngularObjectRegistryListener angularObjectRegistryListener; private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; - DependencyResolver depResolver; + private DependencyResolver depResolver; public InterpreterFactory(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, DependencyResolver depResolver) - throws InterpreterException, IOException { + throws InterpreterException, IOException, RepositoryException { this(conf, new InterpreterOption(true), angularObjectRegistryListener, remoteInterpreterProcessListener, depResolver); } @@ -84,11 +91,12 @@ public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultO AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, DependencyResolver depResolver) - throws InterpreterException, IOException { + throws InterpreterException, IOException, RepositoryException { this.conf = conf; this.defaultOption = defaultOption; this.angularObjectRegistryListener = angularObjectRegistryListener; this.depResolver = depResolver; + this.interpreterRepositories = depResolver.getRepos(); this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS); interpreterClassList = replsConf.split(","); @@ -101,7 +109,7 @@ public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultO init(); } - private void init() throws InterpreterException, IOException { + private void init() throws InterpreterException, IOException, RepositoryException { ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); // Load classes @@ -172,7 +180,11 @@ private void init() throws InterpreterException, IOException { if (found) { // add all interpreters in group - add(groupName, groupName, defaultOption, p); + add(groupName, + groupName, + new LinkedList(), + defaultOption, + p); groupClassNameMap.remove(groupName); break; } @@ -186,7 +198,7 @@ private void init() throws InterpreterException, IOException { logger.info("Interpreter setting group {} : id={}, name={}", setting.getGroup(), settingId, setting.getName()); for (Interpreter interpreter : setting.getInterpreterGroup()) { - logger.info(" className = {}", interpreter.getClassName()); + logger.info(" className = {}", interpreter.getClassName()); } } } @@ -225,12 +237,11 @@ private void loadFromFile() throws IOException { // previously created setting should turn this feature on here. setting.getOption().setRemote(true); - - InterpreterSetting intpSetting = new InterpreterSetting( setting.id(), setting.getName(), setting.getGroup(), + setting.getDependencies(), setting.getOption()); InterpreterGroup interpreterGroup = createInterpreterGroup( @@ -244,8 +255,41 @@ private void loadFromFile() throws IOException { } this.interpreterBindings = info.interpreterBindings; + + if (info.interpreterRepositories != null) { + for (RemoteRepository repo : info.interpreterRepositories) { + if (!depResolver.getRepos().contains(repo)) { + this.interpreterRepositories.add(repo); + } + } + } } + private void loadInterpreterDependencies(InterpreterSetting intSetting) + throws IOException, RepositoryException { + // dependencies to prevent library conflict + File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + intSetting.id()); + if (localRepoDir.exists()) { + FileUtils.cleanDirectory(localRepoDir); + } + + // load dependencies + List deps = intSetting.getDependencies(); + if (deps != null) { + for (Dependency d: deps) { + if (d.getExclusions() != null) { + depResolver.load( + d.getGroupArtifactVersion(), + d.getExclusions(), + conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id()); + } else { + depResolver.load( + d.getGroupArtifactVersion(), + conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id()); + } + } + } + } private void saveToFile() throws IOException { String jsonString; @@ -254,6 +298,7 @@ private void saveToFile() throws IOException { InterpreterInfoSaving info = new InterpreterInfoSaving(); info.interpreterBindings = interpreterBindings; info.interpreterSettings = interpreterSettings; + info.interpreterRepositories = interpreterRepositories; jsonString = gson.toJson(info); } @@ -330,15 +375,21 @@ public List getRegisteredInterpreterList() { * @throws IOException */ public InterpreterGroup add(String name, String groupName, + List dependencies, InterpreterOption option, Properties properties) - throws InterpreterException, IOException { + throws InterpreterException, IOException, RepositoryException { synchronized (interpreterSettings) { InterpreterSetting intpSetting = new InterpreterSetting( name, groupName, + dependencies, option); + if (dependencies.size() > 0) { + loadInterpreterDependencies(intpSetting); + } + InterpreterGroup interpreterGroup = createInterpreterGroup( intpSetting.id(), groupName, option, properties); @@ -354,13 +405,13 @@ private InterpreterGroup createInterpreterGroup(String id, String groupName, InterpreterOption option, Properties properties) - throws InterpreterException , NullArgumentException { + throws InterpreterException, NullArgumentException { //When called from REST API without option we receive NPE - if (option == null ) + if (option == null) throw new NullArgumentException("option"); //When called from REST API without option we receive NPE - if (properties == null ) + if (properties == null) throw new NullArgumentException("properties"); AngularObjectRegistry angularObjectRegistry; @@ -393,7 +444,8 @@ private InterpreterGroup createInterpreterGroup(String id, if (option.isRemote()) { intp = createRemoteRepl(info.getPath(), info.getClassName(), - properties); + properties, + interpreterGroup.id); } else { intp = createRepl(info.getPath(), info.getClassName(), @@ -428,6 +480,9 @@ public void remove(String id) throws IOException { saveToFile(); } } + + File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id); + FileUtils.deleteDirectory(localRepoDir); } /** @@ -511,8 +566,10 @@ public List getNoteInterpreterSettingBinding(String noteId) { * @param properties * @throws IOException */ - public void setPropertyAndRestart(String id, InterpreterOption option, - Properties properties) throws IOException { + public void setPropertyAndRestart(String id, + InterpreterOption option, + Properties properties, + List dependencies) throws IOException, RepositoryException { synchronized (interpreterSettings) { InterpreterSetting intpsetting = interpreterSettings.get(id); if (intpsetting != null) { @@ -523,11 +580,14 @@ public void setPropertyAndRestart(String id, InterpreterOption option, intpsetting.getInterpreterGroup().destroy(); intpsetting.setOption(option); + intpsetting.setDependencies(dependencies); InterpreterGroup interpreterGroup = createInterpreterGroup( intpsetting.id(), intpsetting.getGroup(), option, properties); intpsetting.setInterpreterGroup(interpreterGroup); + + loadInterpreterDependencies(intpsetting); saveToFile(); } else { throw new InterpreterException("Interpreter setting id " + id @@ -661,13 +721,14 @@ private Interpreter createRepl(String dirName, String className, private Interpreter createRemoteRepl(String interpreterPath, String className, - Properties property) { - + Properties property, String interpreterId) { int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); + String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterId; int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE); LazyOpenInterpreter intp = new LazyOpenInterpreter(new RemoteInterpreter( property, className, conf.getInterpreterRemoteRunnerPath(), - interpreterPath, connectTimeout, maxPoolSize, remoteInterpreterProcessListener)); + interpreterPath, localRepoPath, connectTimeout, + maxPoolSize, remoteInterpreterProcessListener)); return intp; } @@ -690,4 +751,19 @@ private URL[] recursiveBuildLibList(File path) throws MalformedURLException { return new URL[] {path.toURI().toURL()}; } } + + public List getRepositories() { + return this.interpreterRepositories; + } + + public void addRepository(String id, String url, boolean snapshot, Authentication auth) + throws IOException { + depResolver.addRepo(id, url, snapshot, auth); + saveToFile(); + } + + public void removeRepository(String id) throws IOException { + depResolver.delRepo(id); + saveToFile(); + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java index ae507d48d1f..786a723f8ea 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java @@ -17,6 +17,8 @@ package org.apache.zeppelin.interpreter; +import org.sonatype.aether.repository.RemoteRepository; + import java.util.List; import java.util.Map; @@ -26,4 +28,5 @@ public class InterpreterInfoSaving { public Map interpreterSettings; public Map> interpreterBindings; + public List interpreterRepositories; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 301ed238fb6..e8080a2ebe7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -17,9 +17,12 @@ package org.apache.zeppelin.interpreter; +import java.util.LinkedList; +import java.util.List; import java.util.Properties; import java.util.Random; +import org.apache.zeppelin.dep.Dependency; import org.apache.zeppelin.notebook.utility.IdHashes; /** @@ -32,21 +35,26 @@ public class InterpreterSetting { private String description; private Properties properties; private InterpreterGroup interpreterGroup; + private List dependencies; private InterpreterOption option; - public InterpreterSetting(String id, String name, + public InterpreterSetting(String id, + String name, String group, + List dependencies, InterpreterOption option) { this.id = id; this.name = name; this.group = group; + this.dependencies = dependencies; this.option = option; } public InterpreterSetting(String name, String group, + List dependencies, InterpreterOption option) { - this(generateId(), name, group, option); + this(generateId(), name, group, dependencies, option); } public String id() { @@ -90,6 +98,17 @@ public Properties getProperties() { return properties; } + public List getDependencies() { + if (dependencies == null) { + return new LinkedList(); + } + return dependencies; + } + + public void setDependencies(List dependencies) { + this.dependencies = dependencies; + } + public InterpreterOption getOption() { if (option == null) { option = new InterpreterOption(); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 17d91ccb86f..8fea69315dd 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -24,17 +24,21 @@ import java.io.File; import java.io.IOException; +import java.util.LinkedList; import java.util.List; import java.util.Properties; import org.apache.commons.lang.NullArgumentException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.Dependency; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; import org.apache.zeppelin.interpreter.mock.MockInterpreter2; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.sonatype.aether.RepositoryException; public class InterpreterFactoryTest { @@ -42,6 +46,7 @@ public class InterpreterFactoryTest { private File tmpDir; private ZeppelinConfiguration conf; private InterpreterContext context; + private DependencyResolver depResolver; @Before public void setUp() throws Exception { @@ -55,7 +60,8 @@ public void setUp() throws Exception { System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); conf = new ZeppelinConfiguration(); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null); + depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); context = new InterpreterContext("note", "id", "title", "text", null, null, null, null, null); } @@ -98,14 +104,14 @@ public void testBasic() { } @Test - public void testFactoryDefaultList() throws IOException { + public void testFactoryDefaultList() throws IOException, RepositoryException { // get default list from default setting List all = factory.getDefaultInterpreterSettingList(); assertEquals(2, all.size()); assertEquals(factory.get(all.get(0)).getInterpreterGroup().getFirst().getClassName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); // add setting - factory.add("a mock", "mock2", new InterpreterOption(false), new Properties()); + factory.add("a mock", "mock2", new LinkedList(), new InterpreterOption(false), new Properties()); all = factory.getDefaultInterpreterSettingList(); assertEquals(2, all.size()); assertEquals("mock1", factory.get(all.get(0)).getName()); @@ -113,16 +119,16 @@ public void testFactoryDefaultList() throws IOException { } @Test - public void testExceptions() throws IOException { + public void testExceptions() throws InterpreterException, IOException, RepositoryException { List all = factory.getDefaultInterpreterSettingList(); // add setting with null option & properties expected nullArgumentException.class try { - factory.add("a mock", "mock2", null, new Properties()); + factory.add("a mock", "mock2", new LinkedList(), null, new Properties()); } catch(NullArgumentException e) { assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage()); } try { - factory.add("a mock" , "mock2" , new InterpreterOption(false),null); + factory.add("a mock", "mock2", new LinkedList(), new InterpreterOption(false), null); } catch (NullArgumentException e){ assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage()); } @@ -130,17 +136,17 @@ public void testExceptions() throws IOException { @Test - public void testSaveLoad() throws InterpreterException, IOException { + public void testSaveLoad() throws IOException, RepositoryException { // interpreter settings assertEquals(2, factory.get().size()); // check if file saved assertTrue(new File(conf.getInterpreterSettingPath()).exists()); - factory.add("newsetting", "mock1", new InterpreterOption(false), new Properties()); + factory.add("newsetting", "mock1", new LinkedList(), new InterpreterOption(false), new Properties()); assertEquals(3, factory.get().size()); - InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null); + InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver); assertEquals(3, factory2.get().size()); } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index 4fa8ef65375..04a6cebea75 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -25,6 +25,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterOption; @@ -40,6 +41,7 @@ public class NoteInterpreterLoaderTest { private File tmpDir; private ZeppelinConfiguration conf; private InterpreterFactory factory; + private DependencyResolver depResolver; @Before public void setUp() throws Exception { @@ -58,7 +60,8 @@ public void setUp() throws Exception { MockInterpreter11.register("mock11", "group1", "org.apache.zeppelin.interpreter.mock.MockInterpreter11"); MockInterpreter2.register("mock2", "group2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null); + depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); } @After diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 506b682b293..539cb2da45d 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -35,6 +35,7 @@ import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterOption; @@ -55,6 +56,7 @@ import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonatype.aether.RepositoryException; public class NotebookTest implements JobListenerFactory{ private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class); @@ -66,6 +68,7 @@ public class NotebookTest implements JobListenerFactory{ private Notebook notebook; private NotebookRepo notebookRepo; private InterpreterFactory factory; + private DependencyResolver depResolver; @Before public void setUp() throws Exception { @@ -86,7 +89,8 @@ public void setUp() throws Exception { MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null); + depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); @@ -161,7 +165,7 @@ public void testReloadAllNotes() throws IOException { } @Test - public void testPersist() throws IOException, SchedulerException{ + public void testPersist() throws IOException, SchedulerException, RepositoryException { Note note = notebook.createNote(); // run with default repl @@ -173,8 +177,8 @@ public void testPersist() throws IOException, SchedulerException{ note.persist(); Notebook notebook2 = new Notebook( - conf, notebookRepo, schedulerFactory, new InterpreterFactory(conf, null, null, null), this, - null); + conf, notebookRepo, schedulerFactory, + new InterpreterFactory(conf, null, null, null, depResolver), this, null); assertEquals(1, notebook2.getAllNotes().size()); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 31970afd133..753fab2f0aa 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -28,6 +28,7 @@ import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterOutput; @@ -57,6 +58,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory { private Notebook notebookSync; private NotebookRepoSync notebookRepoSync; private InterpreterFactory factory; + private DependencyResolver depResolver; private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class); @Before @@ -85,7 +87,8 @@ public void setUp() throws Exception { MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null); + depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); SearchService search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index 2e2801c6635..0d4ff861a78 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -27,6 +27,7 @@ import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; @@ -48,6 +49,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory { private Notebook notebook; private NotebookRepo notebookRepo; private InterpreterFactory factory; + private DependencyResolver depResolver; private File mainZepDir; private File mainNotebookDir; @@ -73,7 +75,8 @@ public void setUp() throws Exception { MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); this.schedulerFactory = new SchedulerFactory(); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null); + depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf);