diff --git a/bin/storage-cli.js b/bin/storage-cli.js index 4ceac252..e9ab46a8 100755 --- a/bin/storage-cli.js +++ b/bin/storage-cli.js @@ -325,5 +325,68 @@ var storage = new StandaloneStorage(config.Storage, function(err) { } ); break; + case 'upgrade_logs': + // upgrade all non-compressed logs to gzip-compressed + // This is part of Cronicle Version 0.5 and can be discarded after that release + var zlib = require('zlib'); + print( "Special Cronicle v0.5 Log Upgrade\n" ); + print( "Compressing all job logs...\n\n" ); + + storage.listEach( 'logs/completed', + function(item, idx, callback) { + var job_path = 'jobs/' + item.id; + var old_path = 'jobs/' + item.id + '/log.txt'; + var new_path = old_path + '.gz'; + + storage.get( job_path, function(err, job) { + if (err) { + // silently skip -- job record deleted or already converted + return callback(); + } + + storage.getStream( old_path, function(err, stream) { + if (err) { + // silently skip -- log deleted or already converted + return callback(); + } + + print( "Compressing: " + old_path + "..." ); + + var gzip = zlib.createGzip( config['gzip_opts'] || {} ); + stream.pipe( gzip ); + + storage.putStream( new_path, gzip, function(err) { + if (err) { + print("Failed to store job log: " + new_path + ": " + err + "\n"); + return callback(); + } + + // delete uncompressed log + storage.delete( old_path, function(err, data) { + if (err) { + print("Failed to delete job log: " + old_path + ": " + err + "\n"); + return callback(); + } + + // set new expiration + var expiration = Math.floor(job.time_end || Tools.timeNow()) + (86400 * (job.log_expire_days || config['job_data_expire_days'])); + var dargs = Tools.getDateArgs( expiration ); + verbose( "<<" + dargs.yyyy_mm_dd + ">>" ); + + storage.expire( new_path, expiration ); + + print( "OK.\n" ); + callback(); + } ); // delete + } ); // putStream + } ); // getStream + } ); // get + }, + function(err) { + print( "\nAll job logs compressed.\nExiting.\n\n"); + } + ); + break; + } // switch }); diff --git a/lib/api/job.js b/lib/api/job.js index 7ec73e82..fffc6079 100644 --- a/lib/api/job.js +++ b/lib/api/job.js @@ -60,14 +60,17 @@ module.exports = Class.create({ id: /^\w+$/ }, callback)) return; - var key = 'jobs/' + args.query.id + '/log.txt'; + var key = 'jobs/' + args.query.id + '/log.txt.gz'; self.storage.getStream( key, function(err, stream) { if (err) { return callback( "404 Not Found", {}, "(No log file found.)\n" ); } - var headers = { 'Content-Type': "text/plain" }; + var headers = { + 'Content-Type': "text/plain", + 'Content-Encoding': "gzip" + }; // optional download instead of view if (args.query.download) { @@ -330,7 +333,7 @@ module.exports = Class.create({ args.session = session; // job log must be available for this to work - self.storage.head( 'jobs/' + params.id + '/log.txt', function(err, info) { + self.storage.head( 'jobs/' + params.id + '/log.txt.gz', function(err, info) { if (err && params.need_log) { return self.doError('job', "Failed to fetch job details: " + err, callback); } diff --git a/lib/job.js b/lib/job.js index 0a9b5ecc..4f021d51 100644 --- a/lib/job.js +++ b/lib/job.js @@ -9,6 +9,7 @@ var os = require('os'); var path = require('path'); var posix = require('posix'); var sqparse = require('shell-quote').parse; +var zlib = require('zlib'); var Class = require("pixl-class"); var Tools = require("pixl-tools"); @@ -1012,7 +1013,7 @@ module.exports = Class.create({ // upload local job log file // or send to storage directly if we're master var self = this; - var path = 'jobs/' + job.id + '/log.txt'; + var path = 'jobs/' + job.id + '/log.txt.gz'; // if we're master, upload directly to storage if (this.multi.master) { @@ -1027,10 +1028,12 @@ module.exports = Class.create({ fs.writeFileSync( job.log_file, data ); } - // get read stream + // get read stream and prepare to compress it var stream = fs.createReadStream( job.log_file ); + var gzip = zlib.createGzip( self.config.get('gzip_opts') || {} ); + stream.pipe( gzip ); - self.storage.putStream( path, stream, function(err) { + self.storage.putStream( path, gzip, function(err) { if (err) { self.logError('storage', "Failed to store job log: " + path + ": " + err); if (callback) callback(err); @@ -1144,7 +1147,7 @@ module.exports = Class.create({ } // set expiration on job log - var log_path = 'jobs/' + job.id + '/log.txt'; + var log_path = 'jobs/' + job.id + '/log.txt.gz'; this.storage.expire( log_path, Tools.timeNow(true) + (86400 * (job.log_expire_days || this.server.config.get('job_data_expire_days'))) ); // add to global activity, event log, and completed events diff --git a/package.json b/package.json index 1f3117f3..bfcdb18a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Cronicle", - "version": "0.2.6", + "version": "0.5.0", "description": "A simple, distributed task scheduler and runner with a web based UI.", "author": "Joseph Huckaby ", "homepage": "https://github.com/jhuckaby/Cronicle",