Version 0.8.25

Removed old garbage collection hack (was from the Node v0.12 days).
Added new api_status call for monitoring purposes.
Disable UDP broadcast server if port is zero or false.
Now keeping track of tick age (exposed in api_status).
If tick age is over 60 and status API is invoked, server will self-restart (should never happen).
Improved logging for crashes and emergency shutdowns.
Added PID log column by default.
Bumped pixl-server version to v1.0.15.

bin/control.sh

@@ -26,7 +26,7 @@ HOMEDIR="$(dirname "$(cd -- "$(dirname "$0")" && (pwd -P 2>/dev/null || pwd))")"
 cd $HOMEDIR
 #
 # the path to your binary, including options if necessary
-BINARY="node --expose_gc --always_compact $HOMEDIR/lib/main.js"
+BINARY="node $HOMEDIR/lib/main.js"
 #
 # the path to your PID file
 PIDFILE=$HOMEDIR/logs/cronicled.pid

bin/debug.sh

@@ -7,4 +7,4 @@
 HOMEDIR="$(dirname "$(cd -- "$(dirname "$0")" && (pwd -P 2>/dev/null || pwd))")"
 
 cd $HOMEDIR
-node --expose_gc --always_compact --trace-warnings $HOMEDIR/lib/main.js --debug --echo "$@"
+node --trace-warnings $HOMEDIR/lib/main.js --debug --echo "$@"

lib/api.js

@@ -57,6 +57,41 @@ module.exports = Class.create({
 		} );
 	},
 
+	api_status: function(args, callback) {
+		// simple status, used by monitoring tools
+		var tick_age = 0;
+		var now = Tools.timeNow();
+		if (this.lastTick) tick_age = now - this.lastTick;
+
+		// do not cache this API response
+		this.forceNoCacheResponse(args);
+
+		var data = {
+			code: 0,
+			version: this.server.__version,
+			node: process.version,
+			hostname: this.server.hostname,
+			ip: this.server.ip,
+			pid: process.pid,
+			now: now,
+			uptime: Math.floor( now - (this.server.started || now) ),
+			last_tick: this.lastTick || now,
+			tick_age: tick_age,
+			cpu: process.cpuUsage(),
+			mem: process.memoryUsage()
+		};
+
+		callback(data);
+
+		// self-check: if tick_age is over 60 seconds, force a self restart
+		if (tick_age > 60) {
+			var msg = "EMERGENCY SHUTDOWN: Tick age is over 60 seconds (" + Math.floor(tick_age) + "s)";
+			this.logger.set('sync', true);
+			this.logDebug(1, msg, data);
+			this.restartLocalServer({ reason: msg });
+		}
+	},
+
 	forceNoCacheResponse: function(args) {
 		// make sure this response isn't cached, ever
 		args.response.setHeader( 'Cache-Control', 'no-cache, no-store, must-revalidate, proxy-revalidate' );

lib/discovery.js

@@ -22,6 +22,12 @@ module.exports = Class.create({
 		this.nearbyServers = {};
 		this.lastDiscoveryBroadcast = 0;
 
+		// disable if port is unset
+		if (!this.server.config.get('udp_broadcast_port')) {
+			if (callback) callback();
+			return;
+		}
+
 		// guess best broadcast IP
 		this.broadcastIP = this.server.config.get('broadcast_ip') || this.calcBroadcastIP();
 		this.logDebug(4, "Using broadcast IP: " + this.broadcastIP );

lib/engine.js

@@ -226,7 +226,8 @@ module.exports = Class.create({
 	tick: function() {
 		// called every second
 		var self = this;
-		var now = Tools.timeNow(true);
+		this.lastTick = Tools.timeNow();
+		var now = Math.floor(this.lastTick);
 
 		if (this.numSocketClients) {
 			var status = {
@@ -585,7 +586,7 @@ module.exports = Class.create({
 				function(hostname, callback) {
 					var server = candidates[hostname];
 
-					var api_url = self.getServerBaseAPIURL( hostname, server.ip ) + '/app/ping';
+					var api_url = self.getServerBaseAPIURL( hostname, server.ip ) + '/app/status';
 					self.logDebug(10, "Sending API request to remote server: " + hostname + ": " + api_url);
 
 					// send request
@@ -602,6 +603,10 @@ module.exports = Class.create({
 							self.logDebug(10, "Failed to ping server: " + hostname + ": " + data.description);
 							return callback();
 						}
+						if (data.tick_age > 60) {
+							self.logDebug(1, "WARNING: Failed to ping server: " + hostname + ": Tick age is " + Math.floor(data.tick_age) + "s", data);
+							return callback();
+						}
 
 						// success, at least one superior ponged our ping
 						// relinquish command to them
@@ -1040,6 +1045,7 @@ module.exports = Class.create({
 		// perform emergency shutdown due to uncaught exception
 		this.logger.set('sync', true);
 		this.logError('crash', "Emergency Shutdown: " + err);
+		this.logDebug(1, "Emergency Shutdown: " + err);
 		this.abortAllLocalJobs();
 	}
 

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "Cronicle",
-	"version": "0.8.24",
+	"version": "0.8.25",
 	"description": "A simple, distributed task scheduler and runner with a web based UI.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/Cronicle",
@@ -51,7 +51,7 @@
 		"pixl-request": "^1.0.21",
 		"pixl-mail": "^1.0.8",
 		"pixl-perf": "^1.0.5",
-		"pixl-server": "^1.0.14",
+		"pixl-server": "^1.0.15",
 		"pixl-server-storage": "^2.0.9",
 		"pixl-server-web": "^1.1.7",
 		"pixl-server-api": "^1.0.1",

sample_conf/config.json

@@ -7,7 +7,7 @@
 
 	"log_dir": "logs",
 	"log_filename": "[component].log",
-	"log_columns": ["hires_epoch", "date", "hostname", "component", "category", "code", "msg", "data"],
+	"log_columns": ["hires_epoch", "date", "hostname", "pid", "component", "category", "code", "msg", "data"],
 	"log_archive_path": "logs/archives/[yyyy]/[mm]/[dd]/[filename]-[yyyy]-[mm]-[dd].log.gz",
 	"log_crashes": true,
 	"copy_job_logs_to": "",