storm-1726: use Put#addColumn to replace the deprecated Put#add

.gitignore

@@ -2,9 +2,7 @@
 /classes
 /lib
 deploy/lib
-deploy/logs
 .emacs-project
-*.jar
 bin/jzmq
 .DS_Store
 deploy/classes
@@ -21,20 +19,38 @@ _release
 *.zip
 *.tar.gz
 .lein-deps-sum
-*.iml
 target
 /.project/
 /.lein-plugins/
-*.ipr
-*.iws
-.idea
 #.*
 !/.travis.yml
 !/.gitignore
 _site
 dependency-reduced-pom.xml
-derby.log
 metastore_db
+build
+/docs/javadocs
+*.class
+
+# logs
+logs
+*.log
+
+# Eclipse
 .settings/
 .project
 .classpath
+
+# Intellij
+*.iml
+*.ipr
+*.iws
+.idea
+
+# Package Files
+*.jar
+*.war
+*.ear
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*

CHANGELOG.md

@@ -1,11 +1,151 @@
 ## 2.0.0
+ * STORM-1681: Bug in scheduling cyclic topologies when scheduling with RAS
+ * STORM-1687: divide by zero in StatsUtil
+ * STORM-1464: storm-hdfs support for multiple file outputs
+ * STORM-515: Clojure documentation and examples
+ * STORM-1279: port backtype.storm.daemon.supervisor to java
+ * STORM-1668: Flux silently fails while setting a non-existent property.
+ * STORM-1271: Port backtype.storm.daemon.task to java
+ * STORM-822: Kafka Spout New Consumer API
+ * STORM-1663: Stats couldn't handle null worker HB.
+ * STORM-1665: Worker cannot instantiate kryo
+ * STORM-1666: Kill from the UI fails silently
+ * STORM-1610: port pacemaker_state_factory_test.clj to java
+ * STORM-1611: port org.apache.storm.pacemaker.pacemaker to java
+ * STORM-1268: port builtin-metrics to java
+ * STORM-1648: drpc spout reconnect on failure
+ * STORM-1631: Storm CGroup bugs
+ * STORM-1616: Add RAS API for Trident
+ * STORM-1623: nimbus.clj's minor bug
+ * STORM-1624: Add maven central status in README
+ * STORM-1232: port backtype.storm.scheduler.DefaultScheduler to java
+ * STORM-1231: port backtype.storm.scheduler.EvenScheduler to java
+ * STORM-1523: util.clj available-port conversion to java
+ * STORM-1252: port backtype.storm.stats to java
+ * STORM-1250: port backtype.storm.serialization-test to java
+ * STORM-1605: use '/usr/bin/env python' to check python version
+ * STORM-1618: Add the option of passing config directory
+ * STORM-1269: port backtype.storm.daemon.common to java
+ * STORM-1270: port drpc to java
+ * STORM-1274: port LocalDRPC to java
+ * STORM-1590: port defmeters/defgauge/defhistogram... to java for all of our code to use
+ * STORM-1529: Change default worker temp directory location for workers
+ * STORM-1543: DRPCSpout should always try to reconnect disconnected DRPCInvocationsClient
+ * STORM-1528: Fix CsvPreparableReporter log directory
+ * STORM-1561: Supervisor should relaunch worker if assignments have changed
+ * STORM-1283: port backtype.storm.MockAutoCred to java
+ * STORM-1592: clojure code calling into Utils.exitProcess throws ClassCastException
+ * STORM-1579: Fix NoSuchFileException when running tests in storm-core
+ * STORM-1244: port backtype.storm.command.upload-credentials to java
+ * STORM-1245: port backtype.storm.daemon.acker to java
+ * STORM-1545: Topology Debug Event Log in Wrong Location
+ * STORM-1254: port ui.helper to java
+ * STORM-1571: Improvment Kafka Spout Time Metric
+ * STORM-1569: Allowing users to specify the nimbus thrift server queue size.
+ * STORM-1564: fix wrong package-info in org.apache.storm.utils.staticmocking
+ * STORM-1267: Port set_log_level
+ * STORM-1266: Port rebalance
+ * STORM-1265: Port monitor
+ * STORM-1572: throw NPE when parsing the command line arguments by CLI
+ * STORM-1273: port backtype.storm.cluster to java
+ * STORM-1479: use a simple implemention for IntSerializer
+ * STORM-1255: port storm_utils.clj to java and split Time tests into its
+ * STORM-1566: Worker exits with error o.a.s.d.worker [ERROR] Error on initialization of server mk-worker
+ * STORM-1558: Utils in java breaks component page due to illegal type cast
+ * STORM-1553: port event.clj to java
+ * STORM-1262: port backtype.storm.command.dev-zookeeper to java.
+ * STORM-1243: port backtype.storm.command.healthcheck to java.
+ * STORM-1246: port backtype.storm.local-state to java.
+ * STORM-1516: Fixed issue in writing pids with distributed cluster mode.
+ * STORM-1253: port backtype.storm.timer to java
+ * STORM-1258: port thrift.clj to Thrift.java
+ * STORM-1336: Evalute/Port JStorm cgroup support and implement cgroup support for RAS
+ * STORM-1511: min/max operators implementation in Trident streams API.
+ * STROM-1263: port backtype.storm.command.kill-topology to java
+ * STORM-1260: port backtype.storm.command.activate to java
+ * STORM-1261: port backtype.storm.command.deactivate to java
+ * STORM-1264: port backtype.storm.command.list to java
+ * STORM-1272: port backtype.storm.disruptor to java
+ * STORM-1248: port backtype.storm.messaging.loader to java
+ * STORM-1538: Exception being thrown after Utils conversion to java
+ * STORM-1242: migrate backtype.storm.command.config-value to java
+ * STORM-1226: Port backtype.storm.util to java
+ * STORM-1436: Random test failure on BlobStoreTest / HdfsBlobStoreImplTest (occasionally killed)
+ * STORM-1476: Filter -c options from args and add them as part of storm.options
+ * STORM-1257: port backtype.storm.zookeeper to java
+ * STORM-1504: Add Serializer and instruction for AvroGenericRecordBolt
+ * STORM-1524: Add Pluggable daemon metrics Reporters
+ * STORM-1521: When using Kerberos login from keytab with multiple bolts/executors ticket is not renewed in hbase bolt.
+
+## 1.0.1
+ * STORM-1704: When logviewer_search.html opens daemon file, next search always show no result
+ * STORM-1714: StatefulBolts ends up as normal bolts while using TopologyBuilder.setBolt without parallelism
+ * STORM-1683: only check non-system streams by default
+ * STORM-1680: Provide configuration to set min fetch size in KafkaSpout
+ * STORM-1649: Optimize Kryo instaces creation in trident windowing
+ * STORM-1696: status not sync if zk fails in backpressure
+ * STORM-1693: Move stats cleanup to executor shutdown
+ * STORM-1670: LocalState#get(String) can throw FileNotFoundException which results in not removing worker heartbeats and supervisor is kind of stuck and goes down after some time.
+ * STORM-1677: Test resource files are excluded from source distribution, which makes logviewer-test failing
+ * STORM-1585: Add DDL support for UDFs in storm-sql
 
 ## 1.0.0
+ * STORM-1671: Enable logviewer to delete a dir without yaml
+ * STORM-1673: log4j2/worker.xml refers old package of LoggerMetricsConsumer
+ * STORM-1669: Fix SolrUpdateBolt flush bug
+ * STORM-1573: Add batch support for MongoInsertBolt
+ * STORM-1660: remove flux gitignore file and move rules to top level gitignore
+ * STORM-1634: Refactoring of Resource Aware Scheduler
+ * STORM-1030: Hive Connector Fixes
+ * STORM-676: Storm Trident support for sliding/tumbling windows
+ * STORM-1630: Add guide page for Windows users
+ * STORM-1655: Flux doesn't set return code to non-zero when there's any exception while deploying topology to remote cluster
+ * STORM-1537: Upgrade to kryo3 in master
+ * STORM-1654: HBaseBolt creates tick tuples with no interval when we don't set flushIntervalSecs
+ * STORM-1625: Move storm-sql dependencies out of lib folder
+ * STORM-1556: nimbus.clj/wait-for-desired-code-replication wrong reset for current-replication-count-jar in local mode
+ * STORM-1636: Supervisor shutdown with worker id pass in being nil
+ * STORM-1602: Blobstore UTs are failed on Windows
+ * STORM-1629: Files/move doesn't work properly with non-empty directory in Windows
+ * STORM-1549: Add support for resetting tuple timeout from bolts via the OutputCollector
+ * STORM-971: Metric for messages lost due to kafka retention
+ * STORM-1483: add storm-mongodb connector
+ * STORM-1608: Fix stateful topology acking behavior
+ * STORM-1609: Netty Client is not best effort delivery on failed Connection
+ * STORM-1620: Update curator to fix CURATOR-209
+ * STORM-1469: Adding Plain Sasl Transport Plugin
+ * STORM-1588: Do not add event logger details if number of event loggers is zero
+ * STORM-1606: print the information of testcase which is on failure
+ * STORM-1601: Check if /backpressure/storm-id node exists before requesting children
+ * STORM-1574: Better handle backpressure exception etc.
+ * STORM-1587: Avoid NPE while prining Metrics
+ * STORM-1570: Storm SQL support for nested fields and array
+ * STORM-1576: fix ConcurrentModificationException in addCheckpointInputs
+ * STORM-1488: UI Topology Page component last error timestamp is from 1970
+ * STORM-1552: Fix topology event sampling log dir
+ * STORM-1542: Remove profile action retry in case of non-zero exit code
+ * STORM-1540: Fix Debug/Sampling for Trident
+ * STORM-1522: REST API throws invalid worker log links
+ * STORM-1541: Change scope of 'hadoop-minicluster' to test
+ * STORM-1532: Fix readCommandLineOpts to parse JSON correctly in windows
+ * STORM-1539: Improve Storm ACK-ing performance
+ * STORM-1519: Storm syslog logging not confirming to RFC5426 3.1
+ * STORM-1520: Nimbus Clojure/Zookeeper issue ("stateChanged" method not found)
+ * STORM-1531: Junit and mockito dependencies need to have correct scope defined in storm-elasticsearch pom.xml
+ * STORM-1526: Improve Storm core performance
+ * STORM-1517: Add peek api in trident stream
+ * STORM-1455: kafka spout should not reset to the beginning of partition when offsetoutofrange exception occurs
+ * STORM-1505: Add map, flatMap and filter functions in trident stream
+ * STORM-1518: Backport of STORM-1504
+ * STORM-1510: Fix broken nimbus log link
+ * STORM-1503: Worker should not crash on failure to send heartbeats to Pacemaker/ZK
+ * STORM-1176: Checkpoint window evaluated/expired state
+ * STORM-1494: Add link to supervisor log in Storm UI
  * STORM-1496: Nimbus periodically throws blobstore-related exception
  * STORM-1484: ignore subproject .classpath & .project file
  * STORM-1478: make bolts getComponentConfiguration method cleaner/simpler
  * STORM-1499: fix wrong package name for storm trident
- * STORM-1463: added file scehma to log4j config files for windows env
+ * STORM-1463: added file schema to log4j config files for windows env
  * STORM-1485: DRPC Connectivity Issues
  * STORM-1486: Fix storm-kafa documentation
  * STORM-1214: add javadoc for Trident Streams and Operations
@@ -250,7 +390,8 @@
  * STORM-1217: making small fixes in RAS
 
 ## 0.10.1
-
+ * STORM-584: Fix logging for LoggingMetricsConsumer metrics.log file
+ * STORM-1596: Do not use single Kerberos TGT instance between multiple threads
  * STORM-1481: avoid Math.abs(Integer) get a negative value
  * STORM-1121: Deprecate test only configuraton nimbus.reassign
  * STORM-1180: FLUX logo wasn't appearing quite right

DEVELOPER.md

@@ -179,7 +179,7 @@ your fork up to date with the latest changes of the upstream (official) `storm`
 
 ### Approve a pull request
 
-[BYLAWS](http://storm.apache.org/documentation/BYLAWS.html) describes the condition of approval for code / non-code change. 
+[BYLAWS](http://storm.apache.org/contribute/BYLAWS.html) describes the condition of approval for code / non-code change. 
 
 Please refer Approvals -> Actions section for more details.
 

README.markdown

@@ -1,4 +1,6 @@
-Master Branch: [![Travis CI](https://travis-ci.org/apache/storm.svg?branch=master)](https://travis-ci.org/apache/storm)
+Master Branch: 
+[![Travis CI](https://travis-ci.org/apache/storm.svg?branch=master)](https://travis-ci.org/apache/storm)
+[![Maven Version](https://maven-badges.herokuapp.com/maven-central/org.apache.storm/storm-core/badge.svg)](http://search.maven.org/#search|gav|1|g:"org.apache.storm"%20AND%20a:"storm-core")
 
 Storm is a distributed realtime computation system. Similar to how Hadoop provides a set of general primitives for doing batch processing, Storm provides a set of general primitives for doing realtime computation. Storm is simple, can be used with any programming language, [is used by many companies](http://storm.apache.org/documentation/Powered-By.html), and is a lot of fun to use!
 
@@ -91,6 +93,8 @@ under the License.
 * Boyang Jerry Peng ([@jerrypeng](https://github.com/jerrypeng))
 * Zhuo Liu ([@zhuoliu](https://github.com/zhuoliu))
 * Haohui Mai ([@haohui](https://github.com/haohui))
+* Sanket Chintapalli ([@redsanket](https://github.com/redsanket))
+* Longda Feng ([@longda](https://github.com/longdafeng))
 
 ## Contributors
 
@@ -249,7 +253,14 @@ under the License.
 * Satish Duggana ([@satishd](https://github.com/satishd))
 * Seth Ammons ([@sethgrid](https://github.com/sethgrid))
 * Aaron Dixon ([@atdixon](https://github.com/atdixon))
-* Roshan Naik([@roshannaik](https://github.com/roshannaik))
+* Roshan Naik ([@roshannaik](https://github.com/roshannaik))
+* John Fang ([@hustfxj](https://github.com/hustfxj))
+* Dan Bahir ([#dbahir](https://github.com/dbahir))
+* Alessandro Bellina ([#abellina](https://github.com/abellina))
+* Basti Liu ([@basti](https://github.com/bastiliu))
+* Jark Wu ([@jark](https://github.com/wuchong))
+* Cody Wang ([@unsleepy22](https://github.com/unsleepy22))
+
 
 ## Acknowledgements
 

bin/flight.bash

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-JDKPATH="/home/y/share/yjava_jdk/java"
+JDKPATH=$JAVA_HOME
 BINPATH="/usr/bin"
 USER=`whoami`
 
@@ -59,7 +59,7 @@ function dump_record {
 
 function jstack_record {
     FILENAME=jstack-$1-${NOW}.txt
-    $BINPATH/jstack $1 > "$2/${FILENAME}"
+    $BINPATH/jstack $1 > "$2/${FILENAME}" 2>&1
 }
 
 function jmap_record {

bin/storm

@@ -30,16 +30,8 @@ while [ -h "${PRG}" ]; do
   fi
 done
 
-# find python >= 2.6
-if [ -a /usr/bin/python2.6 ]; then
-  PYTHON=/usr/bin/python2.6
-fi
-
-if [ -z "$PYTHON" ]; then
-  PYTHON=/usr/bin/python
-fi
-
 # check for version
+PYTHON="/usr/bin/env python"
 majversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f1`
 minversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f2`
 numversion=$(( 10 * $majversion + $minversion))
@@ -51,12 +43,15 @@ fi
 STORM_BIN_DIR=`dirname ${PRG}`
 export STORM_BASE_DIR=`cd ${STORM_BIN_DIR}/..;pwd`
 
-#check to see if the conf dir is given as an optional argument
+#check to see if the conf dir or file is given as an optional argument
 if [ $# -gt 1 ]; then
   if [ "--config" = "$1" ]; then
     conf_file=$2
+    if [ -d "$conf_file" ]; then
+      conf_file=$conf_file/storm.yaml
+    fi
     if [ ! -f "$conf_file" ]; then
-      echo "Error: Cannot find configuration directory: $conf_file"
+      echo "Error: Cannot find configuration file: $conf_file"
       exit 1
     fi
     STORM_CONF_FILE=$conf_file
@@ -71,4 +66,4 @@ if [ -f "${STORM_CONF_DIR}/storm-env.sh" ]; then
   . "${STORM_CONF_DIR}/storm-env.sh"
 fi
 
-exec "$PYTHON" "${STORM_BIN_DIR}/storm.py" "$@"
+exec "${STORM_BIN_DIR}/storm.py" "$@"

bin/storm-config.cmd

@@ -86,6 +86,10 @@ if not defined STORM_LOG_DIR (
 @rem retrieve storm.log4j2.conf.dir from conf file
 @rem
 
+if not defined CMD_TEMP_FILE (
+  set CMD_TEMP_FILE=tmpfile
+)
+
 "%JAVA%" -client -Dstorm.options= -Dstorm.conf.file= -cp "%CLASSPATH%" org.apache.storm.command.config_value storm.log4j2.conf.dir > %CMD_TEMP_FILE%
 
 FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
@@ -125,7 +129,11 @@ FOR /F "delims=" %%i in (%CMD_TEMP_FILE%) do (
 
 
 :storm_opts
- set STORM_OPTS=-Dstorm.options= -Dstorm.home=%STORM_HOME% -Djava.library.path=%JAVA_LIBRARY_PATH%;%JAVA_HOME%\bin;%JAVA_HOME%\lib;%JAVA_HOME%\jre\bin;%JAVA_HOME%\jre\lib
+ if "%set_storm_options%"=="true" (
+  set STORM_OPTS=-Dstorm.options=
+ )
+
+ set STORM_OPTS=%STORM_OPTS% -Dstorm.home=%STORM_HOME% -Djava.library.path=%JAVA_LIBRARY_PATH%;%JAVA_HOME%\bin;%JAVA_HOME%\lib;%JAVA_HOME%\jre\bin;%JAVA_HOME%\jre\lib
  set STORM_OPTS=%STORM_OPTS% -Dlog4j.configurationFile=%STORM_LOG4J2_CONFIGURATION_FILE%
  set STORM_OPTS=%STORM_OPTS% -Dstorm.log.dir=%STORM_LOG_DIR%
  del /F %CMD_TEMP_FILE%