Merge branch 'develop'

Conflicts:
	install-syslog4j-jar.sh
	misc/graylog2.conf
	pom.xml
	src/main/java/org/graylog2/Configuration.java
	src/main/java/org/graylog2/Main.java
	src/main/java/org/graylog2/messagehandlers/common/MessageCounter.java
	src/main/java/org/graylog2/messagehandlers/gelf/ChunkedGELFClientManager.java
	src/main/java/org/graylog2/messagehandlers/syslog/SyslogEventHandler.java
	src/test/java/org/graylog2/messagehandlers/gelf/GELFMessageTest.java

.gitignore

@@ -12,9 +12,13 @@ coverage
 build.xml
 nbactions.xml
 
-
+misc/elasticsearch.yml.2
+misc/elasticsearch.yml.dev
 misc/convert_rsyslog_db_to_mongo.rb
 
+plugin/*/*.jar
+*_gl2plugin.jar
+
 build_script/builds/*
 build_script/logs/*
 

README.markdown

@@ -0,0 +1,9 @@
+# graylog2-server
+
+master: [![Build Status](https://secure.travis-ci.org/Graylog2/graylog2-server.png?branch=master)](http://travis-ci.org/Graylog2/graylog2-server)
+
+develop: [![Build Status](https://secure.travis-ci.org/Graylog2/graylog2-server.png?branch=develop)](http://travis-ci.org/Graylog2/graylog2-server)
+
+This is Graylog2.
+
+Please see http://www.graylog2.org/ for more information and documentation. Thank you.

build_script/build_release.sh

@@ -29,22 +29,34 @@ echo $BUILD_DATE > $BUILD_DIR/build_date
 echo "Copying files ..."
 
 # Copy files.
-cp ../target/*-with-dependencies.jar ../README ../COPYING $BUILD_DIR -r
+cp -R ../target/graylog2-server-jar-with-dependencies.jar ../README.markdown ../COPYING $BUILD_DIR
 
 # Rename jar
-mv $BUILD_DIR/*-with-dependencies.jar $BUILD_DIR/graylog2-server.jar
+mv $BUILD_DIR/graylog2-server-jar-with-dependencies.jar $BUILD_DIR/graylog2-server.jar
 
-# Copy example config file
+# Copy example config files
 cp ../misc/graylog2.conf $BUILD_DIR/graylog2.conf.example
+cp ../misc/elasticsearch.yml $BUILD_DIR/elasticsearch.yml.example
 
 # Copy control script
-cp copy/bin $BUILD_DIR -r
+cp -R copy/bin $BUILD_DIR
+
+# Create empty plugin directories.
+mkdir -p $BUILD_DIR/plugin
+mkdir -p $BUILD_DIR/plugin/filters
+mkdir -p $BUILD_DIR/plugin/outputs
+mkdir -p $BUILD_DIR/plugin/inputs
+mkdir -p $BUILD_DIR/plugin/initializers
+mkdir -p $BUILD_DIR/plugin/transports
+mkdir -p $BUILD_DIR/plugin/alarm_callbacks
+
+mkdir -p $BUILD_DIR/log
 
 cd builds/
 
 # tar it
 echo "Building Tarball ..."
-tar cfz $BUILD_NAME.tar.gz $BUILD_NAME
+gnutar cfz $BUILD_NAME.tar.gz $BUILD_NAME
 rm -rf ./$BUILD_NAME
 
 echo "DONE! Created Graylog2 release $BUILD_NAME on $BUILD_DATE"

build_script/copy/bin/graylog2ctl

@@ -3,35 +3,82 @@
 CMD=$1
 NOHUP=`which nohup`
 
+# resolve links - $0 may be a softlink
+GRAYLOG2CTL="$0"
+
+while [ -h "$GRAYLOG2CTL" ]; do
+  ls=`ls -ld "$GRAYLOG2CTL"`
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    GRAYLOG2CTL="$link"
+  else
+    GRAYLOG2CTL=`dirname "$GRAYLOG2CTL"`/"$link"
+  fi
+done
+
+GRAYLOG2CTL_DIR=`dirname "$GRAYLOG2CTL"`
+GRAYLOG2_SERVER_JAR=graylog2-server.jar
+GRAYLOG2_CONF=/etc/graylog2.conf
+GRAYLOG2_PID=/tmp/graylog2.pid
+LOG_FILE=log/graylog2-server.log
 
 start() {
-	echo "Starting graylog2-server ..."
-	$NOHUP java -jar ../graylog2-server.jar &
+    echo "Starting graylog2-server ..."
+    cd "$GRAYLOG2CTL_DIR/.."
+    $NOHUP java -jar ${GRAYLOG2_SERVER_JAR} -f ${GRAYLOG2_CONF} -p ${GRAYLOG2_PID} >> ${LOG_FILE} &
 }
 
 stop() {
-        PID=`cat /tmp/graylog2.pid`
-	echo "Stopping graylog2-server ($PID) ..."
-        kill $PID
+    PID=`cat ${GRAYLOG2_PID}`
+    echo "Stopping graylog2-server ($PID) ..."
+    if kill $PID; then
+        rm ${GRAYLOG2_PID}
+    fi
 }
 
 restart() {
-	echo "Restarting graylog2-server ..."
-        stop
-        start
+    echo "Restarting graylog2-server ..."
+    stop
+    start
+}
+
+status() {
+    pid=$(get_pid)
+    if [ ! -z $pid ]; then
+        if pid_running $pid; then
+            echo "graylog2-server running as pid $pid"
+            return 0
+        else
+            echo "Stale pid file with $pid - removing..."
+            rm ${GRAYLOG2_PID}
+        fi
+    fi
+
+    echo "graylog2-server not running"
+}
+
+get_pid() {
+    cat ${GRAYLOG2_PID} 2> /dev/null
+}
+
+pid_running() {
+    kill -0 $1 2> /dev/null
 }
 
 case "$CMD" in
-	start)
-		start
-		;;
-	stop)
-		stop
-		;;
-	restart)
-		restart
-		;;
-	*)
-		echo "Usage $0 {start|stop|restart}"
-		RETVAL=1
+    start)
+        start
+        ;;
+    stop)
+        stop
+        ;;
+    restart)
+        restart
+        ;;
+    status)
+        status
+        ;;
+    *)
+        echo "Usage $0 {start|stop|restart|status}"
+        RETVAL=1
 esac

contrib/distro/mac_os_x/org.graylog2.graylog2-server.plist

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>Label</key>
+	<string>org.graylog2.graylog2-server</string>
+	<key>ProgramArguments</key>
+	<array>
+		<string>java</string>
+		<string>-jar</string>
+		<string>/usr/local/Cellar/graylog2-server/0.9.6/graylog2-server.jar</string>
+		<string>-f</string>
+		<string>/usr/local/etc/graylog2.conf</string>
+		<string>-p</string>
+		<string>/tmp/graylog2.pid</string>
+	</array>
+	<key>RunAtLoad</key>
+	<true/>
+	<key>KeepAlive</key>
+	<false/>
+	<key>UserName</key>
+	<string>root</string>
+	<key>WorkingDirectory</key>
+	<string>/usr/local</string>
+	<key>StandardErrorPath</key>
+	<string>/usr/local/var/log/graylog2-server/output.log</string>
+	<key>StandardOutPath</key>
+	<string>/usr/local/var/log/graylog2-server/output.log</string>
+</dict>
+</plist>

contrib/distro/fedora/graylog2-0.9.5p1.spec → contrib/distro/redhat/graylog2-0.9.6.spec

@@ -1,7 +1,7 @@
 Name:		graylog2
-Version:	0.9.5p1
+Version:	0.9.6
 Release:	1%{?dist}
-Summary:	Graylog2 is an open source log management solution that stores your logs in MongoDB.
+Summary:	Graylog2 is an open source log management solution that stores your logs in ElasticSearch.
 Group:		System Environment/Daemons 
 License:	GPLv2
 URL:		http://www.graylog2.org/
@@ -14,7 +14,7 @@ BuildRoot:	%(mktemp -ud %{_tmppath}/%{name}-server-%{version}-%{release}-XXXXXX)
 Requires:	java-1.6.0-openjdk
 
 %description
-Graylog2 is an open source syslog implementation that stores your logs in MongoDB. It consists of a server written in Java that accepts your syslog messages via TCP or UDP and stores it in the database. The second part is a Ruby on Rails web interface that allows you to view the log messages.
+Graylog2 is an open source syslog implementation that stores your logs in ElasticSearch. It consists of a server written in Java that accepts your syslog messages via TCP or UDP and stores it in the database. The second part is a Ruby on Rails web interface that allows you to view the log messages.
 
 
 %prep
@@ -26,18 +26,19 @@ Graylog2 is an open source syslog implementation that stores your logs in MongoD
 
 %install
 rm -rf $RPM_BUILD_ROOT
-%{__install} -p -D -m 0755 %{SOURCE3} %{buildroot}%{_initrddir}/%{name}
-
+# Directories
 %{__install} -p -d -m 0755 %{buildroot}%{_sysconfdir}/%{name}
 %{__install} -p -d -m 0755 %{buildroot}%{_sysconfdir}/%{name}/rules
-%{__install} -p -D -m 0755 %{SOURCE1} %{buildroot}%{_sysconfdir}/%{name}/rules/%{name}.drl
-%{__install} -p -D -m 0755 %{SOURCE2} %{buildroot}%{_sysconfdir}/%{name}/%{name}.conf
-
 %{__install} -p -d -m 0755 %{buildroot}%{_datadir}/%{name}
-%{__install} -p -D -m 0644 %{name}-server.jar %{buildroot}%{_datadir}/%{name}/%{name}-server.jar
-
 %{__install} -p -d -m 0755 %{buildroot}%{_localstatedir}/log/%{name}
 
+# Files
+%{__install} -p -D -m 0755 %{SOURCE3} %{buildroot}%{_initrddir}/%{name}
+%{__install} -p -D -m 0644 %{SOURCE1} %{buildroot}%{_sysconfdir}/%{name}/rules/%{name}.drl
+%{__install} -p -D -m 0644 %{SOURCE2} %{buildroot}%{_sysconfdir}/%{name}/%{name}.conf
+
+%{__install} -p -D -m 0644 %{name}-server.jar %{buildroot}%{_datadir}/%{name}/%{name}-server.jar
+
 
 %clean
 rm -rf $RPM_BUILD_ROOT
@@ -69,5 +70,12 @@ fi
 
 
 %changelog
+* Mon Feb 6 2012 Daniel Aharon <daharon@sazze.com> - 0.9.6
+- Update to 0.9.6
+- Fix permissions for files/dirs.
+
+* Mon May 16 2011 Daniel Aharon <daharon@sazze.com> - 0.9.5sazze1
+- Modified Graylog2-server to better handle multiple rules in streams.
+
 * Mon May 16 2011 Daniel Aharon <daharon@sazze.com> - 0.9.5p1
 - Initial packaging for Fedora.

contrib/distro/redhat/graylog2.conf

@@ -0,0 +1,84 @@
+# On which port (UDP) should we listen for Syslog messages? (Standard: 514)
+syslog_listen_port = 514
+syslog_protocol = udp
+
+# ElasticSearch URL (default: http://localhost:9200/)
+elasticsearch_url = http://localhost:9200/
+elasticsearch_index_name = graylog2
+
+# Always try a reverse DNS lookup instead of parsing hostname from syslog message?
+# force_syslog_rdns = false
+# Set time to NOW if parsing date/time from syslog message failed instead of rejecting it?
+allow_override_syslog_date = true
+
+# MongoDB Configuration
+mongodb_useauth = false
+#mongodb_user = grayloguser
+#mongodb_password = 123
+mongodb_host = localhost
+#mongodb_replica_set = localhost:27017,localhost:27018,localhost:27019
+mongodb_database = graylog2
+mongodb_port = 27017
+
+# Raise this according to the maximum connections your MongoDB server can handle if 
+# you encounter MongoDB connection problems or queuing up messages.
+mongodb_max_connections = 150
+
+# Number of threads allowed to be blocked by MongoDB connections multiplier. 
+# Default: 5
+# If mongodb_max_connections is 150, and mongodb_threads_allowed_to_block_multiplier
+# is 5, then 750 threads can block. More than that and an exception will be thrown.
+# http://api.mongodb.org/java/current/com/mongodb/MongoOptions.html#threadsAllowedToBlockForConnectionMultiplier
+mongodb_threads_allowed_to_block_multiplier = 5
+
+# Graylog2 uses an internal message queue that holds all received messages until they# are indexed. The mq_batch_size parameter defines how many messages are sent
+# to ElasticSearch at once (using a _bulk update: 
+# http://www.elasticsearch.org/guide/reference/api/bulk.html) 
+# The mq_poll_freq parameter controls in which interval (in seconds) the message 
+# batch is sent. Example: If you leave the standard values 
+# (mq_batch_size = 4000, mq_poll_freq = 1), Graylog2 will index 4000 messages every 
+# second. If you have spikes with more than 4000 messages per second, the queue will 
+# start growing until you get under 4000 messages/second again. The queue is FIFO and
+# can grow until you run out of RAM. Note that the queue *only* resists in RAM, so if
+# you set the mq_poll_freq to a high value, you may lose a lot of not yet indexed 
+# messages when the server crashes. Run the server in debug mode 
+# (java -jar graylog2-server.jar --debug) with a |grep '^INFO' to see debug 
+# information about the queue and it's size. 
+# (INFO : org.graylog2.periodical.BulkIndexerThread - About to index max 4000 
+# messages. You have a total of 103 messages in the queue. [freq:1s])
+# You can also monitor the queue size in your graylog2-web-interface.
+mq_batch_size = 4000
+mq_poll_freq = 1
+
+# You can set a maximum size of the message queue. If this size is reached, all new 
+# messages will be rejected until messages are removed/indexed from the queue.
+# 0 = unlimited queue size (default)
+mq_max_size = 0
+
+# Enable realtime collection? This will additionally store information about received
+# messages in a MongoDB collection capped to 50MB size and allows to use realtime 
+# features. It usually is a good idea to keep this enabled because it should not 
+# cause much IO load.
+enable_realtime_collection = true
+
+# Graylog Extended Log Format (GELF)
+use_gelf = true
+gelf_listen_address = 0.0.0.0
+gelf_listen_port = 12201
+
+# Drools Rule File (Use to rewrite incoming log messages)
+# rules_file = /etc/graylog2.d/rules/graylog2.drl
+
+# AMQP
+amqp_enabled = false
+#amqp_subscribed_queues = somequeue1:gelf,somequeue2:gelf,somequeue3:syslog
+#amqp_host = localhost
+#amqp_port = 5672
+#amqp_username = guest
+#amqp_password = guest
+#amqp_virtualhost = /
+
+# Forwarders
+# Timeout in seconds for each connection and read of Logg.ly API when forwarding 
+# messages. Default: 3
+#forwarder_loggly_timeout = 3