diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE new file mode 100644 index 00000000000..09c1b3bfd52 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE @@ -0,0 +1,24 @@ +### What is this PR for? +A few sentences describing the overall goals of the pull request's commits. +First time? Check out the contributing guide - https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md + + +### What type of PR is it? +[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring] + +### Todos +* [ ] - Task + +### What is the Jira issue? +* Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN/ +* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533] + +### How should this be tested? +Outline the steps to test the PR here. + +### Screenshots (if appropriate) + +### Questions: +* Does the licenses files need update? +* Is there breaking changes for older versions? +* Does this needs documentation? diff --git a/.gitignore b/.gitignore index aa5306086d1..e13f0e94d47 100644 --- a/.gitignore +++ b/.gitignore @@ -8,15 +8,29 @@ # interpreter /interpreter/ +# interpreter temp files +spark/derby.log +spark/metastore_db +spark-1.*-bin-hadoop* +.spark-dist +zeppelin-server/derby.log + +lens/lens-cli-hist.log + # conf file conf/zeppelin-env.sh +conf/zeppelin-env.cmd conf/zeppelin-site.xml conf/keystore conf/truststore conf/interpreter.json +conf/notebook-authorization.json +conf/shiro.ini +conf/credentials.json # other generated files spark/dependency-reduced-pom.xml +reports #webapp zeppelin-web/node_modules @@ -26,8 +40,9 @@ zeppelin-web/.sass-cache zeppelin-web/bower_components **nbproject/ **node/ -zeppelin-web/reports/coverage +#R +/r/lib/ # project level /logs/ @@ -45,6 +60,7 @@ zeppelin-web/reports/coverage **/data/ **/build/ **/testing/ +!/testing/ # OS generated files # ###################### @@ -72,6 +88,9 @@ Thumbs.db target/ **/target/ +# Generated by Jekyll +docs/_site/ + *~ \#*\# /.emacs.desktop diff --git a/.travis.yml b/.travis.yml index 0c74edfc55d..9ba565b8586 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,64 +14,85 @@ # limitations under the License. language: java -jdk: - - oraclejdk7 + +sudo: false + +cache: + directories: + - .spark-dist + - ${HOME}/.m2/repository/.cache/maven-download-plugin + +addons: + apt: + sources: + - r-packages-precise + packages: + - r-base-dev + - r-cran-evaluate + - r-cran-base64enc + +matrix: + include: + # Test all modules + - jdk: "oraclejdk7" + env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="" + + # Test spark module for 1.5.2 + - jdk: "oraclejdk7" + env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false" + + # Test spark module for 1.4.1 + - jdk: "oraclejdk7" + env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false" + + # Test spark module for 1.3.1 + - jdk: "oraclejdk7" + env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false" + + # Test spark module for 1.2.2 + - jdk: "oraclejdk7" + env: SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false" + + # Test spark module for 1.1.1 + - jdk: "oraclejdk7" + env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false" + + # Test selenium with spark module for 1.6.1 + - jdk: "oraclejdk7" + env: TEST_SELENIUM="true" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false" before_install: + - "ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin" + - mkdir -p ~/R + - echo 'R_LIBS=~/R' > ~/.Renviron + - R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org', lib='~/R')" + - export R_LIBS='~/R' - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" install: - - mvn package -DskipTests -Phadoop-2.3 -Ppyspark -B + - mvn $BUILD_FLAG $PROFILE -B before_script: - - + - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER + - ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER + - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh + - tail conf/zeppelin-env.sh script: - # spark 1.5 - - mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B - - ./testing/startSparkCluster.sh 1.5.1 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.5.1-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - - ./testing/stopSparkCluster.sh 1.5.1 2.3 - # spark 1.4 - - rm -rf `pwd`/interpreter/spark - - mvn package -DskipTests -Pspark-1.4 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.4.1 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn package -Pspark-1.4 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.4.1 2.3 - # spark 1.3 - - rm -rf `pwd`/interpreter/spark - - mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.3.1 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.3.1 2.3 - # spark 1.2 - - rm -rf `pwd`/interpreter/spark - - mvn package -Pspark-1.2 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.2.1 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.2.1 2.3 -# spark 1.1 - - rm -rf `pwd`/interpreter/spark - - mvn package -Pspark-1.1 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.1.1 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.1.1 2.3 + - mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS + + +after_success: + - echo "Travis exited with ${TRAVIS_TEST_RESULT}" after_failure: + - echo "Travis exited with ${TRAVIS_TEST_RESULT}" - cat target/rat.txt - cat zeppelin-server/target/rat.txt - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out + - cat zeppelin-web/npm-debug.log after_script: - - - -notifications: - slack: - secure: dtIkPwlf5uTun19p9TtPEAFmrLOMK2COE8TL9m8LXX/N2WzJaKYvAnovMObEV6KEgK2oZ+72Cke7eBI+Hp4FmHZ2B7mQI/PNCfRZthI3cc3zVmMd25yvLH9AlCRa2bC6R885z2copvzaoZtLBkHnPa8bUrUkbmRp40qkDPQpgO4= + - ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa69016680f..787e17d3d95 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,31 +1,107 @@ # How to contribute -**Zeppelin** is [Apache2 License](https://github.com/apache/incubator-zeppelin/blob/master/CONTRIBUTING.md) Software. +**Zeppelin** is [Apache2 License](https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md) Software. Contributing to Zeppelin (Source code, Documents, Image, Website) means you agree to the Apache2 License. 1. Make sure your issue is not already in the [Jira issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN) 2. If not, create a ticket describing the change you're proposing in the [Jira issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN) 3. Contribute your patch via Pull Request. +Before you start, please read the [Code of Conduct](http://www.apache.org/foundation/policies/conduct.html) carefully, familiarize yourself with it and refer to it whenever you need it. -## SourceControl Workflow +For those of you who are not familiar with Apache project, understanding [How it works](http://www.apache.org/foundation/how-it-works.html) would be quite helpful. + +## Creating a Pull Request +In order to make the review process easier, please follow this template when making a Pull Request: + +``` +### What is this PR for? +A few sentences describing the overall goals of the pull request's commits. +First time? Check out the contributing guide - https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md + +### What type of PR is it? +[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring] + +### Todos +* [ ] - Task + +### What is the Jira issue? +* Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN/ +* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533] + +### How should this be tested? +Outline the steps to test the PR here. + +### Screenshots (if appropriate) + +### Questions: +* Does the licenses files need update? +* Is there breaking changes for older versions? +* Does this needs documentation? +``` + +## Testing a Pull Request +You can also test and review a particular Pull Request. Here are two useful ways. + +* Using a utility provided from Zeppelin. + + ``` + dev/test_zeppelin_pr.py [# of PR] + ``` + + For example, if you want to test `#513`, then the command will be: + + ``` + dev/test_zeppelin_pr.py 513 + ``` + +* Another way is using [github/hub](https://github.com/github/hub). + + ``` + hub checkout https://github.com/apache/zeppelin/pull/[# of PR] + ``` + +The above two methods will help you test and review Pull Requests. + +## Source Control Workflow Zeppelin follows [Fork & Pull] (https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) model. +## The Review Process + +When a Pull Request is submitted, it is being merged or rejected by following review process. + +* Anybody can be a reviewer and may comment on the change and suggest modifications. +* Reviewer can indicate that a patch looks suitable for merging with a comment such as: "Looks good", "LGTM", "+1". +* At least one indication of suitable for merging (e.g. "LGTM") from committer is required to be merged. +* Pull request is open for 1 or 2 days for potential additional review, unless it's got enough indication of suitable for merging. +* Committer can initiate lazy consensus ("Merge if there is no more discussion") and the code can be merged after certain time (normally 24 hours) when there is no review exists. +* Contributor can ping reviewers (including committer) by commenting 'Ready to review' or suitable indication. + +## Becoming a Committer + +The PPMC adds new committers from the active contributors, based on their contribution to Zeppelin. The qualifications for new committers include: + +1. Sustained contributions: Committers should have a history of constant contributions to Zeppelin. +2. Quality of contributions: Committers more than any other community member should submit simple, well-tested, and well-designed patches. +3. Community involvement: Committers should have a constructive and friendly attitude in all community interactions. They should also be active on the dev, user list and reviewing patches. Also help new contributors and users. + + ## Setting up -Here are some things you will need to build and test Zeppelin. +Here are some things you will need to build and test Zeppelin. -### Software Configuration Management(SCM) +### Software Configuration Management (SCM) -Zeppelin uses Git for it's SCM system. Hosted by github.com. `https://github.com/apache/incubator-zeppelin` You'll need git client installed in your development machine. +Zeppelin uses Git for its SCM system. `http://git.apache.org/zeppelin.git` you'll need git client installed in your development machine. +For write access, `https://git-wip-us.apache.org/repos/asf/zeppelin.git` -### Integrated Development Environment(IDE) +### Integrated Development Environment (IDE) You are free to use whatever IDE you prefer, or your favorite command line editor. ### Project Structure Zeppelin project is based on Maven. Maven works by convention & defines [directory structure] (https://maven.apache.org/guides/introduction/introduction-to-the-standard-directory-layout.html) for a project. -The top level pom.xml describes the basic project structure. Currently Zeppelin has the following modules. +The top-level pom.xml describes the basic project structure. Currently Zeppelin has the following modules. zeppelin-interpreter zeppelin-zengine @@ -34,7 +110,6 @@ The top level pom.xml describes the basic project structure. Currently Zeppelin angular shell hive - tajo flink ignite lens @@ -43,13 +118,16 @@ The top level pom.xml describes the basic project structure. Currently Zeppelin zeppelin-server zeppelin-distribution +### Web Project Contribution Guidelines +If you plan on making a contribution to Zeppelin's WebApplication, +please check [its own contribution guidelines](https://github.com/apache/zeppelin/blob/master/zeppelin-web/CONTRIBUTING.md) ### Code convention We are following Google Code style: -* [Java style](http://google-styleguide.googlecode.com/svn/trunk/javaguide.html) -* [Shell style](https://google-styleguide.googlecode.com/svn/trunk/shell.xml) +* [Java style](https://google.github.io/styleguide/javaguide.html) +* [Shell style](https://google.github.io/styleguide/shell.xml) -Checkstyle report location are in `${submodule}/target/site/checkstyle.html` +Check style report location are in `${submodule}/target/site/checkstyle.html` Test coverage report location are in `${submodule}/target/site/cobertura/index.html` #### Build Tools @@ -59,26 +137,31 @@ To build the code, install * Apache Maven ## Getting the source code -First of all, you need the Zeppelin source code. The official location for Zeppelin is [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin) +First of all, you need the Zeppelin source code. The official location for Zeppelin is [http://git.apache.org/zeppelin.git](http://git.apache.org/zeppelin.git). ### git access Get the source code on your development machine using git. ``` -git clone git@github.com:apache/incubator-zeppelin.git zeppelin +git clone git://git.apache.org/zeppelin.git zeppelin ``` -You may also want to develop against a specific release. For example, for branch-0.1 +You may also want to develop against a specific branch. For example, for branch-0.5.6 ``` -git clone -b branch-0.1 git@github.com:apache/incubator-zeppelin.git zeppelin +git clone -b branch-0.5.6 git://git.apache.org/zeppelin.git zeppelin ``` +or with write access + +``` +git clone https://git-wip-us.apache.org/repos/asf/zeppelin.git +``` ### Fork repository -If you want not only build Zeppelin but also make change, then you need fork Zeppelin repository and make pull request. +If you want not only build Zeppelin but also make change, then you need fork Zeppelin github mirror repository (https://github.com/apache/zeppelin) and make pull request. ## Build @@ -105,18 +188,18 @@ Each new File should have its own accompanying unit tests. Each new interpreter Zeppelin has 3 types of tests: - 1. Unit Tests: The unit tests run as part of each package's build. E.g SparkInterpeter Module's unit test is SparkInterpreterTest - 2. Integration Tests: The intergration tests run after all modules are build. The integration tests launch an instance of Zeppelin server. ZeppelinRestApiTest is an example integration test. - 3. GUI integration tests: These tests validate the Zeppelin UI elements. These tests require a running Zepplein server and lauches a web browser to validate Notebook UI elements like Notes and their execution. See ZeppelinIT as an example. + 1. Unit Tests: The unit tests run as part of each package's build. E.g. SparkInterpeter Module's unit test is SparkInterpreterTest + 2. Integration Tests: The integration tests run after all modules are build. The integration tests launch an instance of Zeppelin server. ZeppelinRestApiTest is an example integration test. + 3. GUI integration tests: These tests validate the Zeppelin UI elements. These tests require a running Zeppelin server and launches a web browser to validate Notebook UI elements like Notes and their execution. See ZeppelinIT as an example. -Currently the GUI integration tests are not run in the Maven and are only run in the CI environment when the pull request is submitted to github. Make sure to watch the [CI results] (https://travis-ci.org/apache/incubator-zeppelin/pull_requests) for your pull request. +Currently the GUI integration tests are not run in the Maven and are only run in the CI environment when the pull request is submitted to github. Make sure to watch the [CI results] (https://travis-ci.org/apache/zeppelin/pull_requests) for your pull request. ## Continuous Integration -Zeppelin uses Travis for CI. In the project root there is .travis.yml that configures CI and [publishes CI results] (https://travis-ci.org/apache/incubator-zeppelin/builds) +Zeppelin uses Travis for CI. In the project root there is .travis.yml that configures CI and [publishes CI results] (https://travis-ci.org/apache/zeppelin/builds) -## Run Zepplin server in development mode +## Run Zeppelin server in development mode ``` cd zeppelin-server @@ -135,9 +218,12 @@ Server will be run on http://localhost:8080 ## JIRA Zeppelin manages it's issues in Jira. [https://issues.apache.org/jira/browse/ZEPPELIN](https://issues.apache.org/jira/browse/ZEPPELIN) +## Where to Start +You can find issues for [beginner](https://issues.apache.org/jira/browse/ZEPPELIN-924?jql=project%20%3D%20ZEPPELIN%20and%20status%20%3D%20Open%20and%20labels%20in%20\(beginner%2C%20newbie\)) + ## Stay involved -Everyone is welcome to join our mailling list: +Everyone is welcome to join our mailing list: - * [users@zeppelin.incubator.apache.org](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-users/) is for usage questions, help, and announcements [ [subscribe](mailto:users-subscribe@zeppelin.incubator.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:users-unsubscribe@zeppelin.incubator.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-users/) ] - * [dev@zeppelin.incubator.apache.org](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-users/) is for people who want to contribute code to Zeppelin.[ [subscribe](mailto:dev-subscribe@zeppelin.incubator.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.incubator.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-dev/) ] - * [commits@zeppelin.incubator.apache.org](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-commits/) is for commit messages and patches to Zeppelin. [ [subscribe](mailto:commits-subscribe@zeppelin.incubator.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:commits-unsubscribe@zeppelin.incubator.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/incubator-zeppelin-commits/) ] \ No newline at end of file + * [users@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-users/) is for usage questions, help, and announcements [ [subscribe](mailto:users-subscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:users-unsubscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/zeppelin-users/) ] + * [dev@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-users/) is for people who want to contribute code to Zeppelin.[ [subscribe](mailto:dev-subscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:dev-unsubscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/zeppelin-dev/) ] + * [commits@zeppelin.apache.org](http://mail-archives.apache.org/mod_mbox/zeppelin-commits/) is for commit messages and patches to Zeppelin. [ [subscribe](mailto:commits-subscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20subscribe), [unsubscribe](mailto:commits-unsubscribe@zeppelin.apache.org?subject=send%20this%20email%20to%20unsubscribe), [archive](http://mail-archives.apache.org/mod_mbox/zeppelin-commits/) ] diff --git a/DEPLOY.md b/DEPLOY.md deleted file mode 100644 index 50b1baff91d..00000000000 --- a/DEPLOY.md +++ /dev/null @@ -1,55 +0,0 @@ -Publish Zeppelin Distribution Package ------- - -A. Build package - -Following command will create package zeppelin-VERSION.tar.gz under _zeppelin-distribution/target_ directory. - - mvn clean package -P build-distr - - -B. Upload to S3 bucket ~~web server~~ - - ~~scp zeppelin-distribution/target/zeppelin-VERSION.tar.gz root@www.nflabs.com:/var/www/html/pub/zeppelin/~~ - mvn package -P publish-distr - - -C. Edit www.zeppelin-project.org - -Edit download page to have link to new release. - - -Publish javadoc -------- - -Generate javadoc with following command - - mvn javadoc:javadoc - mv "zeppelin-zengine/target/site/apidocs" "ZEPPELIN_HOMEPAGE/docs/zengine-api/VERSION" - -and publish the web. - - -Publish Maven artifact ------------- - -**Publish to snapshot repository** - - mvn -DperformRelease=true deploy - - -**Publish to release repository** - - mvn -DperformRelease=true release:clean - mvn -DperformRelease=true release:prepare - mvn -DperformRelease=true release:perform - -Artifact is now in staging repository. -Connect https://oss.sonatype.org/ , select staging repository and click "close" -> "release" will finally release it. - - -**Reference** - -https://docs.sonatype.org/display/Repository/How+To+Generate+PGP+Signatures+With+Maven - -https://docs.sonatype.org/display/Repository/Sonatype+OSS+Maven+Repository+Usage+Guide#SonatypeOSSMavenRepositoryUsageGuide-1a.TermsofServiceRepository%3ACentralRepositoryTermsofServiceforSubmitters diff --git a/DISCLAIMER b/DISCLAIMER deleted file mode 100644 index 727300e5d7d..00000000000 --- a/DISCLAIMER +++ /dev/null @@ -1,12 +0,0 @@ -Apache Zeppelin is an effort undergoing incubation at the Apache Software -Foundation (ASF), sponsored by the Apache Incubator PMC. - -Incubation is required of all newly accepted projects until a further -review indicates that the infrastructure, communications, and decision -making process have stabilized in a manner consistent with other -successful ASF projects. - -While incubation status is not necessarily a reflection of the -completeness or stability of the code, it does indicate that the -project has yet to be fully endorsed by the ASF. - diff --git a/LICENSE b/LICENSE index 31af032a404..67a5b7934ae 100644 --- a/LICENSE +++ b/LICENSE @@ -210,68 +210,58 @@ subcomponents is subject to the terms and conditions of the following licenses. -For the PostgreSQL JDBC driver jar file: - Copyright (c) 1997-2011, PostgreSQL Global Development Group - All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - 3. Neither the name of the PostgreSQL Global Development Group nor the names - of its contributors may be used to endorse or promote products derived - from this software without specific prior written permission. +======================================================================== + The Open Font License +======================================================================== - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. +The following components are provided under the Open Font License. See project link for details. +The text of each license is also included at licenses/LICENSE-[project]-[version].txt. + (OFT 1.1) Font Awesome v4.2.0 (http://fortawesome.github.io/Font-Awesome/) - http://scripts.sil.org/OFL -This product contains webfonts from 'Font Awesome', which can be obtains at: - * LICENSE: - * SIL OFL 1.1 (http://scripts.sil.org/OFL) - * HOMEPAGE: - * http://fortawesome.github.io/Font-Awesome/ +======================================================================== +MIT licenses +======================================================================== +The following components are provided under the MIT License. See project link for details. +The text of each license is also included at licenses/LICENSE-[project]-[version].txt. + (The MIT License) Font Awesome (stylesheet) 4.2.0 (http://fortawesome.github.io/Font-Awesome/) - https://fortawesome.github.io/Font-Awesome/license/ + (The MIT License) Simple line icons v1.0.0 (http://thesabbir.github.io/simple-line-icons/) - https://github.com/thesabbir/simple-line-icons/tree/1.0.0 + (The MIT License) jekyll-bootstrap 0.3.0 (https://github.com/plusjade/jekyll-bootstrap) - https://github.com/plusjade/jekyll-bootstrap + (The MIT License) jekyll 1.3.0 (http://jekyllrb.com/) - https://github.com/jekyll/jekyll/blob/v1.3.0/LICENSE -This product contains a stylesheet from 'Font Awesome', which can be obtains at: - * LICENSE: - * MIT License (http://opensource.org/licenses/mit-license.html) - * HOMEPAGE: - * http://fortawesome.github.io/Font-Awesome/ +======================================================================== +Apache licenses +======================================================================== +The following components are provided under the Apache License. See project link for details. +The text of each license is also included at licenses/LICENSE-[project]-[version].txt. -This product contains embeddable code editor 'Ace', which can be obtains at: + (Apache 2.0) Bootstrap v3.0.2 (http://getbootstrap.com/) - https://github.com/twbs/bootstrap/blob/v3.0.2/LICENSE - * LICENSE: - * BSD (https://github.com/ajaxorg/ace/blob/master/LICENSE) - Copyright (c) 2010, Ajax.org B.V. - All rights reserved. - * HOMEPAGE: - * http://ace.c9.io/ +======================================================================== +BSD 3-Clause licenses +======================================================================== +The following components are provided under the BSD 3-Clause license. See file headers and project links for details. + (BSD 3 Clause) portions of rscala 1.0.6 (https://dahl.byu.edu/software/rscala/) - https://cran.r-project.org/web/packages/rscala/index.html + r/R/rzeppelin/R/{common.R, globals.R,protocol.R,rServer.R,scalaInterpreter.R,zzz.R } + r/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/{Package.scala, RClient.scala} + (BSD 3 Clause) portions of Scala (http://www.scala-lang.org/download) - http://www.scala-lang.org/download/#License + r/src/main/scala/scala/Console.scala -This product contains 'Simple line icons', which can be obtains at: +======================================================================== +BSD 2-Clause licenses +======================================================================== +The following components are provided under the BSD 3-Clause license. See file headers and project links for details. - * LICENSE: - * MIT License (http://opensource.org/licenses/mit-license.html) - * HOMEPAGE: - * http://thesabbir.github.io/simple-line-icons/ + (BSD 2 Clause) portions of SQLLine (http://sqlline.sourceforge.net/) - http://sqlline.sourceforge.net/#license + jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java \ No newline at end of file diff --git a/NOTICE b/NOTICE index 5b6d272b994..bfbb6271795 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ -Apache Zeppelin (incubating) -Copyright 2015 The Apache Software Foundation +Apache Zeppelin +Copyright 2015 - 2016 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/README.md b/README.md index b2beaa4c090..131463bd5ba 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ -#Zeppelin +#Zeppelin -**Documentation:** [User Guide](http://zeppelin.incubator.apache.org/docs/index.html)
-**Mailing List:** [User and Dev mailing list](http://zeppelin.incubator.apache.org/community.html)
-**Continuous Integration:** [![Build Status](https://secure.travis-ci.org/apache/incubator-zeppelin.png?branch=master)](https://travis-ci.org/apache/incubator-zeppelin)
-**Contributing:** [Contribution Guide](https://github.com/apache/incubator-zeppelin/blob/master/CONTRIBUTING.md)
-**License:** [Apache 2.0](https://github.com/apache/incubator-zeppelin/blob/master/LICENSE) +**Documentation:** [User Guide](http://zeppelin.apache.org/docs/latest/index.html)
+**Mailing Lists:** [User and Dev mailing list](http://zeppelin.apache.org/community.html)
+**Continuous Integration:** [![Build Status](https://secure.travis-ci.org/apache/zeppelin.png?branch=master)](https://travis-ci.org/apache/zeppelin)
+**Contributing:** [Contribution Guide](https://github.com/apache/zeppelin/blob/master/CONTRIBUTING.md)
+**Issue Tracker:** [Jira](https://issues.apache.org/jira/browse/ZEPPELIN)
+**License:** [Apache 2.0](https://github.com/apache/zeppelin/blob/master/LICENSE) **Zeppelin**, a web-based notebook that enables interactive data analytics. You can make beautiful data-driven, interactive and collaborative documents with SQL, Scala and more. @@ -14,147 +15,276 @@ Core feature: * Built-in Apache Spark support -To know more about Zeppelin, visit our web site [http://zeppelin.incubator.apache.org](http://zeppelin.incubator.apache.org) +To know more about Zeppelin, visit our web site [http://zeppelin.apache.org](http://zeppelin.apache.org) ## Requirements + * Git * Java 1.7 - * Tested on Mac OSX, Ubuntu 14.X, CentOS 6.X + * Tested on Mac OSX, Ubuntu 14.X, CentOS 6.X, Windows 7 Pro SP1 * Maven (if you want to build from the source code) - * Node.js Package Manager + * Node.js Package Manager (npm, downloaded by Maven during build phase) ## Getting Started ### Before Build -If you don't have requirements prepared, install it. +If you don't have requirements prepared, install it. (The installation method may vary according to your environment, example is for Ubuntu.) + ``` sudo apt-get update -sudo apt-get install openjdk-7-jdk sudo apt-get install git -sudo apt-get install maven +sudo apt-get install openjdk-7-jdk sudo apt-get install npm sudo apt-get install libfontconfig ``` -### Build -If you want to build Zeppelin from the source, please first clone this repository. And then: -``` -mvn clean package -DskipTests -``` - -Build with specific Spark version +#### Proxy settings (optional) +If you are behind a corporate Proxy with NTLM authentication you can use [Cntlm Authentication Proxy](http://cntlm.sourceforge.net/) . -Spark 1.5.x -``` -mvn clean package -Pspark-1.5 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests -``` -Spark 1.4.x -``` -mvn clean package -Pspark-1.4 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests +Before build start, run these commands from shell. ``` -Spark 1.3.x +export http_proxy=http://localhost:3128 +export https_proxy=http://localhost:3128 +export HTTP_PROXY=http://localhost:3128 +export HTTPS_PROXY=http://localhost:3128 +npm config set proxy http://localhost:3128 +npm config set https-proxy http://localhost:3128 +npm config set registry "http://registry.npmjs.org/" +npm config set strict-ssl false +npm cache clean +git config --global http.proxy http://localhost:3128 +git config --global https.proxy http://localhost:3128 +git config --global url."http://".insteadOf git:// ``` -mvn clean package -Pspark-1.3 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests -``` -Spark 1.2.x -``` -mvn clean package -Pspark-1.2 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests -``` -Spark 1.1.x -``` -mvn clean package -Pspark-1.1 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests + +After build is complete, run these commands to cleanup. ``` -CDH 5.X +npm config rm proxy +npm config rm https-proxy +git config --global --unset http.proxy +git config --global --unset https.proxy +git config --global --unset url."http://".insteadOf ``` -mvn clean package -Pspark-1.2 -Dhadoop.version=2.5.0-cdh5.3.0 -Phadoop-2.4 -DskipTests + +_Notes:_ + - If you are on Windows replace `export` with `set` to set env variables + - Replace `localhost:3128` with standard pattern `http://user:pwd@host:port` + - Git configuration is needed because Bower use it for fetching from GitHub + +#### Install maven ``` -Yarn (Hadoop 2.7.x) +wget http://www.eu.apache.org/dist/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz +sudo tar -zxf apache-maven-3.3.3-bin.tar.gz -C /usr/local/ +sudo ln -s /usr/local/apache-maven-3.3.3/bin/mvn /usr/local/bin/mvn ``` -mvn clean package -Pspark-1.4 -Dspark.version=1.4.1 -Dhadoop.version=2.7.0 -Phadoop-2.6 -Pyarn -DskipTests + +_Notes:_ + - Ensure node is installed by running `node --version` + - Ensure maven is running version 3.1.x or higher with `mvn -version` + - Configure maven to use more memory than usual by `export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=1024m"` + +### Build +If you want to build Zeppelin from the source, please first clone this repository, then: + ``` -Yarn (Hadoop 2.6.x) +mvn clean package -DskipTests [Options] ``` -mvn clean package -Pspark-1.1 -Dhadoop.version=2.6.0 -Phadoop-2.6 -Pyarn -DskipTests + +Each Interpreter requires different Options. + + +#### Spark Interpreter + +To build with a specific Spark version, Hadoop version or specific features, define one or more of the following profiles and options: + +##### `-Pspark-[version]` + +Set spark major version + +Available profiles are + ``` -Yarn (Hadoop 2.4.x) +-Pspark-1.6 +-Pspark-1.5 +-Pspark-1.4 +-Pspark-1.3 +-Pspark-1.2 +-Pspark-1.1 +-Pcassandra-spark-1.5 +-Pcassandra-spark-1.4 +-Pcassandra-spark-1.3 +-Pcassandra-spark-1.2 +-Pcassandra-spark-1.1 ``` -mvn clean package -Pspark-1.1 -Dhadoop.version=2.4.0 -Phadoop-2.4 -Pyarn -DskipTests + +minor version can be adjusted by `-Dspark.version=x.x.x` + + +##### `-Phadoop-[version]` + +set hadoop major version + +Available profiles are + ``` -Yarn (Hadoop 2.3.x) +-Phadoop-0.23 +-Phadoop-1 +-Phadoop-2.2 +-Phadoop-2.3 +-Phadoop-2.4 +-Phadoop-2.6 ``` -mvn clean package -Pspark-1.1 -Dhadoop.version=2.3.0 -Phadoop-2.3 -Pyarn -DskipTests + +minor version can be adjusted by `-Dhadoop.version=x.x.x` + +##### `-Pyarn` (optional) + +enable YARN support for local mode +> YARN for local mode is not supported for Spark v1.5.0 or higher. Set `SPARK_HOME` instead. + +##### `-Ppyspark` (optional) + +enable [PySpark](http://spark.apache.org/docs/latest/api/python/) support for local mode. + +##### `-Pr` (optional) + +enable [R](https://www.r-project.org/) support with [SparkR](https://spark.apache.org/docs/latest/sparkr.html) integration. + +##### `-Psparkr` (optional) + +another [R](https://www.r-project.org/) support with [SparkR](https://spark.apache.org/docs/latest/sparkr.html) integration as well as local mode support. + +##### `-Pvendor-repo` (optional) + +enable 3rd party vendor repository (cloudera) + + +##### `-Pmapr[version]` (optional) + +For the MapR Hadoop Distribution, these profiles will handle the Hadoop version. As MapR allows different versions of Spark to be installed, you should specify which version of Spark is installed on the cluster by adding a Spark profile (`-Pspark-1.2`, `-Pspark-1.3`, etc.) as needed. +The correct Maven artifacts can be found for every version of MapR at http://doc.mapr.com + +Available profiles are + ``` -Yarn (Hadoop 2.2.x) +-Pmapr3 +-Pmapr40 +-Pmapr41 +-Pmapr50 +-Pmapr51 ``` -mvn clean package -Pspark-1.1 -Dhadoop.version=2.2.0 -Phadoop-2.2 -Pyarn -DskipTests + + +Here're some examples: + +```sh +# basic build +mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark + +# spark-cassandra integration +mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests + +# with CDH +mvn clean package -Pspark-1.5 -Dhadoop.version=2.6.0-cdh5.5.0 -Phadoop-2.6 -Pvendor-repo -DskipTests + +# with MapR +mvn clean package -Pspark-1.5 -Pmapr50 -DskipTests ``` -Ignite (1.1.0-incubating and later) + +#### Ignite Interpreter + +```sh +mvn clean package -Dignite.version=1.6.0 -DskipTests ``` -mvn clean package -Dignite.version=1.1.0-incubating -DskipTests + +#### Scalding Interpreter + +```sh +mvn clean package -Pscalding -DskipTests ``` ### Configure If you wish to configure Zeppelin option (like port number), configure the following files: + ``` ./conf/zeppelin-env.sh ./conf/zeppelin-site.xml ``` -(You can copy ```./conf/zeppelin-env.sh.template``` into ```./conf/zeppelin-env.sh```. -Same for ```zeppelin-site.xml```.) + +(You can copy `./conf/zeppelin-env.sh.template` into `./conf/zeppelin-env.sh`. +Same for `zeppelin-site.xml`.) #### Setting SPARK_HOME and HADOOP_HOME -Without SPARK_HOME and HADOOP_HOME, Zeppelin uses embedded Spark and Hadoop binaries that you have specified with mvn build option. -If you want to use system provided Spark and Hadoop, export SPARK_HOME and HADOOP_HOME in zeppelin-env.sh +Without `SPARK_HOME` and `HADOOP_HOME`, Zeppelin uses embedded Spark and Hadoop binaries that you have specified with mvn build option. +If you want to use system provided Spark and Hadoop, export `SPARK_HOME` and `HADOOP_HOME` in `zeppelin-env.sh`. You can use any supported version of spark without rebuilding Zeppelin. -``` +```sh # ./conf/zeppelin-env.sh export SPARK_HOME=... export HADOOP_HOME=... ``` #### External cluster configuration + Mesos - # ./conf/zeppelin-env.sh - export MASTER=mesos://... - export ZEPPELIN_JAVA_OPTS="-Dspark.executor.uri=/path/to/spark-*.tgz" or SPARK_HOME="/path/to/spark_home" - export MESOS_NATIVE_LIBRARY=/path/to/libmesos.so - +```sh +# ./conf/zeppelin-env.sh +export MASTER=mesos://... +export ZEPPELIN_JAVA_OPTS="-Dspark.executor.uri=/path/to/spark-*.tgz" or SPARK_HOME="/path/to/spark_home" +export MESOS_NATIVE_LIBRARY=/path/to/libmesos.so +``` + If you set `SPARK_HOME`, you should deploy spark binary on the same location to all worker nodes. And if you set `spark.executor.uri`, every worker can read that file on its node. Yarn - # ./conf/zeppelin-env.sh - export SPARK_HOME=/path/to/spark_dir +```sh +# ./conf/zeppelin-env.sh +export SPARK_HOME=/path/to/spark_dir +``` ### Run - ./bin/zeppelin-daemon.sh start - browse localhost:8080 in your browser. +```sh +./bin/zeppelin-daemon.sh start +``` +And browse [localhost:8080](localhost:8080) in your browser. -For configuration details check __./conf__ subdirectory. + +For configuration details check __`./conf`__ subdirectory. ### Package -To package final distribution do: +To package the final distribution including the compressed archive, run: - mvn clean package -P build-distr +```sh +mvn clean package -Pbuild-distr +``` -The archive is generated under _zeppelin-distribution/target_ directory +To build a distribution with specific profiles, run: -###Run end-to-end tests -Zeppelin comes with a set of end-to-end acceptance tests driving headless selenium browser +```sh +mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4 -Pyarn -Ppyspark +``` - #assumes zeppelin-server running on localhost:8080 (use -Durl=.. to override) - mvn verify +The profiles `-Pspark-1.5 -Phadoop-2.4 -Pyarn -Ppyspark` can be adjusted if you wish to build to a specific spark versions, or omit support such as `yarn`. - #or take care of starting\stoping zeppelin-server from packaged _zeppelin-distribuion/target_ - mvn verify -P using-packaged-distr +The archive is generated under _`zeppelin-distribution/target`_ directory +###Run end-to-end tests +Zeppelin comes with a set of end-to-end acceptance tests driving headless selenium browser +```sh +# assumes zeppelin-server running on localhost:8080 (use -Durl=.. to override) +mvn verify + +# or take care of starting/stoping zeppelin-server from packaged zeppelin-distribuion/target +mvn verify -P using-packaged-distr +``` -[![Analytics](https://ga-beacon.appspot.com/UA-45176241-4/apache/incubator-zeppelin/README.md?pixel)](https://github.com/igrigorik/ga-beacon) +[![Analytics](https://ga-beacon.appspot.com/UA-45176241-4/apache/zeppelin/README.md?pixel)](https://github.com/igrigorik/ga-beacon) diff --git a/SECURITY-README.md b/SECURITY-README.md new file mode 100644 index 00000000000..f9f6645ff5b --- /dev/null +++ b/SECURITY-README.md @@ -0,0 +1,40 @@ + + +# Shiro Authentication +To connect to Zeppelin, users will be asked to enter their credentials. Once logged, a user has access to all notes including other users notes. +This a a first step toward full security as implemented by this pull request (https://github.com/apache/zeppelin/pull/53). + +# Security setup +1. Secure the HTTP channel: Comment the line "/** = anon" and uncomment the line "/** = authcBasic" in the file conf/shiro.ini. Read more about he shiro.ini file format at the following URL http://shiro.apache.org/configuration.html#Configuration-INISections. +2. Secure the Websocket channel : Set to property "zeppelin.anonymous.allowed" to "false" in the file conf/zeppelin-site.xml. You can start by renaming conf/zeppelin-site.xml.template to conf/zeppelin-site.xml +3. Start Zeppelin : bin/zeppelin.sh +4. point your browser to http://localhost:8080 +5. Login using one of the user/password combinations defined in the conf/shiro.ini file. + +# Implementation notes +## Vocabulary +username, owner and principal are used interchangeably to designate the currently authenticated user +## What are we securing ? +Zeppelin is basically a web application that spawn remote interpreters to run commands and return HTML fragments to be displayed on the user browser. +The scope of this PR is to require credentials to access Zeppelin. To achieve this, we use Apache Shiro. +## HTTP Endpoint security +Apache Shiro sits as a servlet filter between the browser and the exposed services and handles the required authentication without any programming required. (See Apache Shiro for more info). +## Websocket security +Securing the HTTP endpoints is not enough, since Zeppelin also communicates with the browser through websockets. To secure this channel, we take the following approach: + 1. The browser on startup requests a ticket through HTTP + 2. The Apache Shiro Servlet filter handles the user auth + 3. Once the user is authenticated, a ticket is assigned to this user and the ticket is returned to the browser + +All websockets communications require the username and ticket to be submitted by the browser. Upon receiving a websocket message, the server checks that the ticket received is the one assigned to the username through the HTTP request (step 3 above). diff --git a/alluxio/pom.xml b/alluxio/pom.xml new file mode 100644 index 00000000000..fa5df776778 --- /dev/null +++ b/alluxio/pom.xml @@ -0,0 +1,200 @@ + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.6.0-SNAPSHOT + .. + + + org.apache.zeppelin + zeppelin-alluxio + jar + 0.6.0-SNAPSHOT + Zeppelin: Alluxio interpreter + http://www.apache.org + + + 1.0.0 + 1.6.1 + + + + org.apache.zeppelin + zeppelin-interpreter + ${project.version} + provided + + + + com.google.guava + guava + + + + org.alluxio + alluxio-shell + ${alluxio.version} + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + + + junit + junit + test + + + + org.mockito + mockito-all + 1.10.8 + test + + + + org.powermock + powermock-api-mockito + ${powermock.version} + test + + + + org.powermock + powermock-core + ${powermock.version} + test + + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + + org.powermock + powermock-reflect + ${powermock.version} + test + + + + org.alluxio + alluxio-core-server + ${alluxio.version} + test + + + + org.alluxio + alluxio-minicluster + ${alluxio.version} + test + + + + org.alluxio + alluxio-underfs-local + ${alluxio.version} + test + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + maven-enforcer-plugin + 1.3.1 + + + enforce + none + + + + + + maven-dependency-plugin + 2.8 + + + copy-dependencies + package + + copy-dependencies + + + ${project.build.directory}/../../interpreter/alluxio + false + false + true + runtime + + + + copy-artifact + package + + copy + + + ${project.build.directory}/../../interpreter/alluxio + false + false + true + runtime + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${project.packaging} + + + + + + + + + + diff --git a/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java b/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java new file mode 100644 index 00000000000..acc59bdcc8a --- /dev/null +++ b/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java @@ -0,0 +1,254 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.alluxio; + +import java.io.IOException; +import java.io.PrintStream; +import java.io.ByteArrayOutputStream; +import java.util.*; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import alluxio.Configuration; +import alluxio.shell.AlluxioShell; + +/** + * Alluxio interpreter for Zeppelin. + */ +public class AlluxioInterpreter extends Interpreter { + + Logger logger = LoggerFactory.getLogger(AlluxioInterpreter.class); + + protected static final String ALLUXIO_MASTER_HOSTNAME = "alluxio.master.hostname"; + protected static final String ALLUXIO_MASTER_PORT = "alluxio.master.port"; + + private AlluxioShell fs; + + private int totalCommands = 0; + private int completedCommands = 0; + + private final String alluxioMasterHostname; + private final String alluxioMasterPort; + + protected final List keywords = Arrays.asList("cat", "chgrp", + "chmod", "chown", "copyFromLocal", "copyToLocal", "count", + "createLineage", "deleteLineage", "du", "fileInfo", "free", + "getCapacityBytes", "getUsedBytes", "listLineages", "load", + "loadMetadata", "location", "ls", "mkdir", "mount", "mv", + "persist", "pin", "report", "rm", "setTtl", "tail", "touch", + "unmount", "unpin", "unsetTtl"); + + public AlluxioInterpreter(Properties property) { + super(property); + + alluxioMasterHostname = property.getProperty(ALLUXIO_MASTER_HOSTNAME); + alluxioMasterPort = property.getProperty(ALLUXIO_MASTER_PORT); + } + + static { + Interpreter.register("alluxio", "alluxio", + AlluxioInterpreter.class.getName(), + new InterpreterPropertyBuilder() + .add(ALLUXIO_MASTER_HOSTNAME, "localhost", "Alluxio master hostname") + .add(ALLUXIO_MASTER_PORT, "19998", "Alluxio master port") + .build()); + } + + @Override + public void open() { + logger.info("Starting Alluxio shell to connect to " + alluxioMasterHostname + + " on port " + alluxioMasterPort); + + System.setProperty(ALLUXIO_MASTER_HOSTNAME, alluxioMasterHostname); + System.setProperty(ALLUXIO_MASTER_PORT, alluxioMasterPort); + fs = new AlluxioShell(new Configuration()); + } + + @Override + public void close() { + logger.info("Closing Alluxio shell"); + + try { + fs.close(); + } catch (IOException e) { + logger.error("Cannot close connection", e); + } + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + String[] lines = splitAndRemoveEmpty(st, "\n"); + return interpret(lines, context); + } + + private InterpreterResult interpret(String[] commands, InterpreterContext context) { + boolean isSuccess = true; + totalCommands = commands.length; + completedCommands = 0; + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos); + PrintStream old = System.out; + + System.setOut(ps); + + for (String command : commands) { + int commandResuld = 1; + String[] args = splitAndRemoveEmpty(command, " "); + if (args.length > 0 && args[0].equals("help")) { + System.out.println(getCommandList()); + } else { + commandResuld = fs.run(args); + } + if (commandResuld != 0) { + isSuccess = false; + break; + } else { + completedCommands += 1; + } + System.out.println(); + } + + System.out.flush(); + System.setOut(old); + + if (isSuccess) { + return new InterpreterResult(Code.SUCCESS, baos.toString()); + } else { + return new InterpreterResult(Code.ERROR, baos.toString()); + } + } + + private String[] splitAndRemoveEmpty(String st, String splitSeparator) { + String[] voices = st.split(splitSeparator); + ArrayList result = new ArrayList(); + for (String voice : voices) { + if (!voice.trim().isEmpty()) { + result.add(voice); + } + } + return result.toArray(new String[result.size()]); + } + + private String[] splitAndRemoveEmpty(String[] sts, String splitSeparator) { + ArrayList result = new ArrayList(); + for (String st : sts) { + result.addAll(Arrays.asList(splitAndRemoveEmpty(st, splitSeparator))); + } + return result.toArray(new String[result.size()]); + } + + @Override + public void cancel(InterpreterContext context) { } + + @Override + public FormType getFormType() { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) { + return completedCommands * 100 / totalCommands; + } + + @Override + public List completion(String buf, int cursor) { + String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " "); + String lastWord = ""; + if (words.length > 0) { + lastWord = words[ words.length - 1 ]; + } + ArrayList voices = new ArrayList(); + for (String command : keywords) { + if (command.startsWith(lastWord)) { + voices.add(command); + } + } + return voices; + } + + private String getCommandList() { + StringBuilder sb = new StringBuilder(); + sb.append("Commands list:"); + sb.append("\n\t[help] - List all available commands."); + sb.append("\n\t[cat ] - Prints the file's contents to the console."); + sb.append("\n\t[chgrp [-R] ] - Changes the group of a file or directory " + + "specified by args. Specify -R to change the group recursively."); + sb.append("\n\t[chmod -R ] - Changes the permission of a file or directory " + + "specified by args. Specify -R to change the permission recursively."); + sb.append("\n\t[chown -R ] - Changes the owner of a file or directory " + + "specified by args. Specify -R to change the owner recursively."); + sb.append("\n\t[copyFromLocal ] - Copies a file or a directory from " + + "local filesystem to Alluxio filesystem."); + sb.append("\n\t[copyToLocal ] - Copies a file or a directory from the " + + "Alluxio filesystem to the local filesystem."); + sb.append("\n\t[count ] - Displays the number of files and directories matching " + + "the specified prefix."); + sb.append("\n\t[createLineage " + + "[ ...]] - Creates a lineage."); + sb.append("\n\t[deleteLineage ] - Deletes a lineage. If " + + "cascade is specified as true, dependent lineages will also be deleted."); + sb.append("\n\t[du ] - Displays the size of the specified file or directory."); + sb.append("\n\t[fileInfo ] - Displays all block info for the specified file."); + sb.append("\n\t[free ] - Removes the file or directory(recursively) " + + "from Alluxio memory space."); + sb.append("\n\t[getCapacityBytes] - Gets the capacity of the Alluxio file system."); + sb.append("\n\t[getUsedBytes] - Gets number of bytes used in the Alluxio file system."); + sb.append("\n\t[listLineages] - Lists all lineages."); + sb.append("\n\t[load ] - Loads a file or directory in Alluxio space, makes it " + + "resident in memory."); + sb.append("\n\t[loadMetadata ] - Loads metadata for the given Alluxio path from the " + + "under file system."); + sb.append("\n\t[location ] - Displays the list of hosts storing the specified file."); + sb.append("\n\t[ls [-R] ] - Displays information for all files and directories " + + "directly under the specified path. Specify -R to display files and " + + "directories recursively."); + sb.append("\n\t[mkdir [path2] ... [pathn]] - Creates the specified directories, " + + "including any parent directories that are required."); + sb.append("\n\t[mount ] - Mounts a UFS path onto an Alluxio path."); + sb.append("\n\t[mv ] - Renames a file or directory."); + sb.append("\n\t[persist ] - Persists a file or directory currently stored " + + "only in Alluxio to the UnderFileSystem."); + sb.append("\n\t[pin ] - Pins the given file or directory in memory (works " + + "recursively for directories). Pinned files are never evicted from memory, unless " + + "TTL is set."); + sb.append("\n\t[report ] - Reports to the master that a file is lost."); + sb.append("\n\t[rm [-R] ] - Removes the specified file. Specify -R to remove file or " + + "directory recursively."); + sb.append("\n\t[setTtl