Version 0.5.

All job logs are now compressed when stored.
BREAKING CHANGE: This changes the internal format of all logs to gzip, and will require a one-time script run to preserve existing logs:
Run this command once on the master server: sudo /opt/cronicle/bin/storage-cli.js upgrade_logs

bin/storage-cli.js

@@ -325,5 +325,68 @@ var storage = new StandaloneStorage(config.Storage, function(err) {
 			} );
 		break;
 
+		case 'upgrade_logs':
+			// upgrade all non-compressed logs to gzip-compressed
+			// This is part of Cronicle Version 0.5 and can be discarded after that release
+			var zlib = require('zlib');
+			print( "Special Cronicle v0.5 Log Upgrade\n" );
+			print( "Compressing all job logs...\n\n" );
+
+			storage.listEach( 'logs/completed', 
+				function(item, idx, callback) {
+					var job_path = 'jobs/' + item.id;
+					var old_path = 'jobs/' + item.id + '/log.txt';
+					var new_path = old_path + '.gz';
+
+					storage.get( job_path, function(err, job) {
+						if (err) {
+							// silently skip -- job record deleted or already converted
+							return callback();
+						}
+
+						storage.getStream( old_path, function(err, stream) {
+							if (err) {
+								// silently skip -- log deleted or already converted
+								return callback();
+							}
+
+							print( "Compressing: " + old_path + "..." );
+
+							var gzip = zlib.createGzip( config['gzip_opts'] || {} );
+							stream.pipe( gzip );
+
+							storage.putStream( new_path, gzip, function(err) {
+								if (err) {
+									print("Failed to store job log: " + new_path + ": " + err + "\n");
+									return callback();
+								}
+
+								// delete uncompressed log
+								storage.delete( old_path, function(err, data) {
+									if (err) {
+										print("Failed to delete job log: " + old_path + ": " + err + "\n");
+										return callback();
+									}
+
+									// set new expiration
+									var expiration = Math.floor(job.time_end || Tools.timeNow()) + (86400 * (job.log_expire_days || config['job_data_expire_days']));
+									var dargs = Tools.getDateArgs( expiration );
+									verbose( "<<" + dargs.yyyy_mm_dd + ">>" );
+
+									storage.expire( new_path, expiration );
+
+									print( "OK.\n" );
+									callback();
+								} ); // delete
+							} ); // putStream
+						} ); // getStream
+					} ); // get
+				}, 
+				function(err) {
+					print( "\nAll job logs compressed.\nExiting.\n\n");
+				}
+			);
+		break;
+
 	} // switch
 });

lib/api/job.js

@@ -60,14 +60,17 @@ module.exports = Class.create({
 			id: /^\w+$/
 		}, callback)) return;
 
-		var key = 'jobs/' + args.query.id + '/log.txt';
+		var key = 'jobs/' + args.query.id + '/log.txt.gz';
 
 		self.storage.getStream( key, function(err, stream) {
 			if (err) {
 				return callback( "404 Not Found", {}, "(No log file found.)\n" );
 			}
 
-			var headers = { 'Content-Type': "text/plain" };
+			var headers = {
+				'Content-Type': "text/plain",
+				'Content-Encoding': "gzip"
+			};
 
 			// optional download instead of view
 			if (args.query.download) {
@@ -330,7 +333,7 @@ module.exports = Class.create({
 			args.session = session;
 
 			// job log must be available for this to work
-			self.storage.head( 'jobs/' + params.id + '/log.txt', function(err, info) {
+			self.storage.head( 'jobs/' + params.id + '/log.txt.gz', function(err, info) {
 				if (err && params.need_log) {
 					return self.doError('job', "Failed to fetch job details: " + err, callback);
 				}

lib/job.js

@@ -9,6 +9,7 @@ var os = require('os');
 var path = require('path');
 var posix = require('posix');
 var sqparse = require('shell-quote').parse;
+var zlib = require('zlib');
 
 var Class = require("pixl-class");
 var Tools = require("pixl-tools");
@@ -1012,7 +1013,7 @@ module.exports = Class.create({
 		// upload local job log file
 		// or send to storage directly if we're master
 		var self = this;
-		var path = 'jobs/' + job.id + '/log.txt';
+		var path = 'jobs/' + job.id + '/log.txt.gz';
 
 		// if we're master, upload directly to storage
 		if (this.multi.master) {
@@ -1027,10 +1028,12 @@ module.exports = Class.create({
 					fs.writeFileSync( job.log_file, data );
 				}
 
-				// get read stream
+				// get read stream and prepare to compress it
 				var stream = fs.createReadStream( job.log_file );
+				var gzip = zlib.createGzip( self.config.get('gzip_opts') || {} );
+				stream.pipe( gzip );
 
-				self.storage.putStream( path, stream, function(err) {
+				self.storage.putStream( path, gzip, function(err) {
 					if (err) {
 						self.logError('storage', "Failed to store job log: " + path + ": " + err);
 						if (callback) callback(err);
@@ -1144,7 +1147,7 @@ module.exports = Class.create({
 		}
 
 		// set expiration on job log
-		var log_path = 'jobs/' + job.id + '/log.txt';
+		var log_path = 'jobs/' + job.id + '/log.txt.gz';
 		this.storage.expire( log_path, Tools.timeNow(true) + (86400 * (job.log_expire_days || this.server.config.get('job_data_expire_days'))) );
 
 		// add to global activity, event log, and completed events

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "Cronicle",
-	"version": "0.2.6",
+	"version": "0.5.0",
 	"description": "A simple, distributed task scheduler and runner with a web based UI.",
 	"author": "Joseph Huckaby <[email protected]>",
 	"homepage": "https://github.com/jhuckaby/Cronicle",