server-generated session_id; client generated crc32 values for chunks;

backend_file_storage_handler.lua

@@ -10,39 +10,75 @@ local ngx = ngx
 local string = string
 local concat = table.concat
 local error = error
-
+local os = os
 
 module(...)
 
 local function end_backend(self, ctx)
   -- last chunk commited?
   if ctx.range_to + 1 == ctx.range_total then
+
+    -- seed body arguments into table..
+    local body_args = {
+            size = ctx.range_total,
+            id = ctx.id,
+            path = ctx.file_path,
+            name = ctx.get_name(),
+            file_name = ctx.file_name,
+            checksum = ctx.checksum,
+            sha1 = ctx.sha1
+    }
+
+    -- parameterize misc value if populated..
+    -- note: misc must be a string of uri-style parameters. i.e. a=1&b=2
+    -- fucking nginx strips inbound complex key->value tables into simple string arrays
+    if ctx.misc ~= nil then
+      for kv in string.gmatch(ctx.misc, "[^&]+") do
+        for k, v in string.gmatch(kv, "(%w.+)=(%w.+)") do
+          body_args[k] = v
+        end
+      end
+    end
+
     ngx.req.set_header('Content-Type', 'application/x-www-form-urlencoded')
     return ngx.location.capture(self.backend, {
         method = ngx.HTTP_POST,
-        body = ngx.encode_args({
-          size = ctx.range_total,
-          id = ctx.id,
-          path = ctx.file_path,
-          name = ctx.get_name(),
-          checksum = ctx.checksum,
-          sha1 = ctx.sha1
-        })
+        body = ngx.encode_args(body_args)
     })
   end
 end
 
+local function move_file(self, ctx)
+  if ctx.file_path and ctx.success_destination_dir then
+    local ret, err = os.rename(ctx.file_path, concat({ctx.success_destination_dir, ctx.file_name}, "/"))
+    if ret == nil or err ~= nil then
+      return string.format("Failed to move completed file: %s to: %s. Error: %s", ctx.file_path, ctx.success_destination_dir, err)
+    end
+  end
+end
+
 -- override
 local function on_body_start(self, ctx)
   local file_path = concat({self.dir, ctx.id}, "/")
+  if ctx.file_name ~= nil then
+    -- use explicitly specified filename..
+    file_path = concat({self.dir, ctx.file_name}, "/")
+  end
   ctx.file_path = file_path
   return self:init_file(ctx)
 end
 
 -- override
 local function on_body_end(self, ctx)
   self:close_file()
-  -- call backend if finished
+  if ctx.range_to + 1 == ctx.range_total then
+    local ret = move_file(self, ctx)
+    if ret then
+      ngx.log(ngx.ERR, ret)
+      return ngx.ERROR
+    end
+  end
+  -- call backend if finished & move successful
   local res = end_backend(self, ctx)
   return {201, string.format("0-%d/%d", ctx.range_to, ctx.range_total), response = res }
 end

big-upload.lua

@@ -4,7 +4,8 @@
 local config = {
     package_path = ngx.var.package_path,
     bu_checksum = ('on' ==  ngx.var.bu_checksum),
-    bu_sha1 = ('on' == ngx.var.bu_sha1)
+    bu_sha1 = ('on' == ngx.var.bu_sha1),
+    bu_mime = ('on' == ngx.var.bu_mime)
 }
 
 if config.package_path then
@@ -15,6 +16,7 @@ local file_storage_handler = require "file_storage_handler"
 local backend_file_storage_handler = require "backend_file_storage_handler"
 local crc32 = require('crc32')
 local sha1= require('sha1_handler')
+local mime = require('mime')
 
 local function report_result(info)
   if type(info) == "table" then
@@ -62,7 +64,10 @@ if config.bu_checksum then
   table.insert(handlers, crc32.handler())
 end
 if config.bu_sha1 then
- table.insert(handlers, sha1.handler(ngx.var.file_storage_path))
+  table.insert(handlers, sha1.handler(ngx.var.file_storage_path))
+end
+if config.bu_mime then
+  table.insert(handlers, mime.handler())
 end
 table.insert(handlers, storage_handler)
 

crc32.lua

@@ -2,21 +2,20 @@
 
 -- CRC32 checksum
 
-local ffi = require('ffi')
 local tonumber = tonumber
 local string = string
 local ngx = ngx
 local table = table
 
+
+local zlib = require ('zlib')
+
 module(...)
 
-local zlib = ffi.load('z')
-ffi.cdef[[
-    unsigned long crc32(unsigned long crc, const char *buf, unsigned len );
-]]
 
 function crc32(data, lastnum)
-  return tonumber(zlib.crc32(lastnum, data, #data))
+  return zlib.crc32()(data, lastnum)
+  -- return zlib.crc32()(lastnum, data, #data)
 end
 
 function validhex(crchex) return #crchex <= 8 and string.match(crchex, "^%x+$") end
@@ -32,6 +31,7 @@ function handler()
   return {
     on_body_start = function (self, ctx)
       ctx.current_checksum = ctx.last_checksum and tonumber(ctx.last_checksum, 16) or ( ctx.first_chunk and 0 )
+
       -- stop checksum processing if X-Last-Checksum is not present for non first chunk
       if not ctx.current_checksum then
         self.on_body = nil
@@ -40,12 +40,13 @@ function handler()
     end,
 
     on_body = function (self, ctx, body)
-      ctx.current_checksum = crc32(body, ctx.current_checksum)
+        ctx.current_checksum = crc32(body, ctx.current_checksum)
     end,
 
     on_body_end = function (self, ctx)
       if ctx.checksum then
         if tonumber(ctx.checksum,16) ~= ctx.current_checksum then
+          ngx.log(ngx.ERR, string.format("Chunk checksum mismatch client=[%s] server=[%s]", ctx.checksum, tohex(ctx.current_checksum)))
           return {400, string.format("Chunk checksum mismatch client=[%s] server=[%s]", ctx.checksum, tohex(ctx.current_checksum))}
         end
       else

example/html/index.html

@@ -22,14 +22,17 @@ <h1>Example Chunked Uploader</h1>
 
 <script>
 
-function doUpload(url, extraParams, sessionID, file, progress, success) {
+function doUpload(url, extraParams, file, progress, success) {
     var chunkSize = 32000,// this is first chunk size, which will be adaptively expanded depending on upload speed
         aborting = false,
         TO = null,
         lastChecksum = null,
         offset = 0,
-        lastChunkTime = 4000; //used for adaptive chunk size recalculation
+        lastChunkTime = 4000, //used for adaptive chunk size recalculation
+        calculateCRCs = true,
+        crcBlockSize = 4096;
     var xhr;
+    var sessionID;
 
     function adjustChunkSize(startTime) {
         var end = new Date().getTime();
@@ -48,6 +51,36 @@ <h1>Example Chunked Uploader</h1>
         }
     }
 
+	var crc32 = (function() {
+
+		var table = new Uint32Array(256);
+
+		// Pre-generate crc32 polynomial lookup table
+		// http://wiki.osdev.org/CRC32#Building_the_Lookup_Table
+		// ... Actually use Alex's because it generates the correct bit order
+		//     so no need for the reversal function
+		for (var i=256; i--;) {
+			var tmp = i;
+			for(var k=8; k--;)
+			{
+				tmp = tmp & 1 ? 3988292384 ^ tmp >>> 1 : tmp >>> 1;
+			}
+			table[i] = tmp;
+		}
+
+		// crc32b
+		// Example input        : [97, 98, 99, 100, 101] (Uint8Array)
+		// Example output       : 2240272485 (Uint32)
+		return function (data) {
+			var crc = -1; // Begin with all bits set ( 0xffffffff )
+			for(var i=0, l=data.length; i<l; i++) {
+				crc = crc >>> 8 ^ table[ crc & 255 ^ data[i] ];
+			}
+			return (crc ^ -1) >>> 0; // Apply binary NOT
+		};
+
+	})();
+
     var uploadNextChunk = function() {
             TO = null;
             var chunkStartTime = new Date().getTime();
@@ -88,13 +121,15 @@ <h1>Example Chunked Uploader</h1>
                 }
 
                 if (xhr.readyState >= 4) {
+
                     if (xhr.status === 200) {
                         progress((chunkEnd + 1) / file.size);
 
                         // done
                         success(xhr.responseText);
 
                     } else if (xhr.status === 201) {
+	                    sessionID = xhr.getResponseHeader('X-Session-Id');
                         offset = chunkEnd + 1;
                         progress(offset / file.size);
                         adjustChunkSize(chunkStartTime);
@@ -130,13 +165,25 @@ <h1>Example Chunked Uploader</h1>
             xhr.setRequestHeader('Content-Type', 'application/octet-stream');
             xhr.setRequestHeader('Content-Disposition', 'attachment; filename="' + encodeURIComponent(file.name) + '"');
             xhr.setRequestHeader('X-Content-Range', 'bytes ' + chunkStart + '-' + chunkEnd + '/' + file.size);
-            xhr.setRequestHeader('X-Session-ID', sessionID);
+            if (sessionID != null) {
+                xhr.setRequestHeader('X-Session-ID', sessionID);
+            }
             if(lastChecksum) {
                 //client must pass the checksum of all previous chunks
                 //this way server will continue checksum calculation
                 xhr.setRequestHeader('X-Last-Checksum', lastChecksum);
             }
-            xhr.send(currentBlob);
+            if (calculateCRCs) {
+				blobToArrayBuffer(currentBlob, function (arrayBuffer) {
+					var byteArray = new Uint8Array(arrayBuffer);
+					var crc = createCRC(byteArray);
+					xhr.setRequestHeader('X-Checksum', crc.toString(16));
+					//console.log('X-Checksum', crc + " " + crc.toString(16));
+					xhr.send(currentBlob);
+				});
+			} else {
+				xhr.send(currentBlob);
+			}
         };
 
     TO = setTimeout(uploadNextChunk, 1);
@@ -161,6 +208,32 @@ <h1>Example Chunked Uploader</h1>
             uploadNextChunk();
         }
     };
+
+	function blobToArrayBuffer (blob, callback) {
+		var fileReader     = new FileReader();
+		fileReader.onload  = function() {
+			callback(this.result);
+		};
+		fileReader.readAsArrayBuffer(blob)
+	}
+
+	function createCRC (byteArray) {
+		var crcBlockCount = Math.ceil(byteArray.length / crcBlockSize);
+		var crc = 0;
+		var crcBlockStart = 0;
+		for (var i=0; i<crcBlockCount; i++) {
+			var crcBlockEnd = crcBlockStart + crcBlockSize;
+			if (crcBlockEnd > byteArray.length) {
+				crcBlockEnd = byteArray.length;
+			}
+			var crcBlock = byteArray.subarray(crcBlockStart, crcBlockEnd);
+			crc = crc32(crcBlock);
+			//console.log("crc's: " + i + ": " + crc + " crcBlock.length: " + crcBlock.length);
+			crcBlockStart = crcBlockEnd;
+		}
+		return crc;
+	}
+
 }
 
 
@@ -183,10 +256,7 @@ <h1>Example Chunked Uploader</h1>
 
         var file = document.getElementById('file-to-upload').files[0];
 
-        //generate random long number for SessionID
-        var sessionID = Math.round(Math.pow(10,17)*Math.random());
-
-        doUpload('/upload', 'test=1', sessionID, file, function(progress) {
+        doUpload('/upload', 'test=1', file, function(progress) {
 
             console.log('Total file progress is ' + Math.floor(progress * 100) + '%');
 

file_storage_handler.lua

@@ -9,6 +9,7 @@
 local setmetatable = setmetatable
 local concat = table.concat
 local io = io
+local os = os
 local string = string
 local error = error
 local ngx = ngx
@@ -56,8 +57,21 @@ local function close_file(self)
   end
 end
 
+local function move_file(self, ctx)
+  if ctx.file_path and ctx.success_destination_dir then
+    local ret, err = os.rename(ctx.file_path, concat({ctx.success_destination_dir, ctx.file_name}, "/"))
+    if ret == nil or err ~= nil then
+      return string.format("Failed to move completed file: %s to: %s. Error: %s", ctx.file_path, ctx.success_destination_dir, err)
+    end
+  end
+end
+
 local function on_body_start(self, ctx)
   ctx.file_path = concat({self.dir, ctx.id}, "/")
+  if ctx.file_name ~= nil then
+    -- use explicitly specified filename..
+    file_path = concat({self.dir, ctx.file_name}, "/")
+  end
   return self:init_file(ctx)
 end
 
@@ -74,6 +88,13 @@ end
 
 local function on_body_end(self, ctx)
   close_file(self)
+  if ctx.range_to + 1 == ctx.range_total then
+    local ret = move_file(self, ctx)
+    if ret then
+      ngx.log(ngx.ERR, ret)
+      return ngx.ERROR
+    end
+  end
   -- return what what we have on server
   return {201, string.format("0-%d/%d", ctx.range_to, ctx.range_total) }
 end