Allow read buffer size tuning & add readCallback example

resources/examples/partialReadCallback.py

@@ -0,0 +1,20 @@
+def queueRequests(target, wordlists):
+    global engine
+    engine = RequestEngine(endpoint=target.endpoint,
+                           concurrentConnections=2,
+                           readCallback=handleRead,
+                           readSize=256, # TCP socket buffer size - the server may choose to send less
+                           )
+    engine.start()
+
+    engine.queue(target.req)
+
+# data is *just* the last socket read contents
+# so if you're really unlucky your token might get split over two reads
+def handleRead(data):
+    if 'token' in data:
+        engine.queue('something-using-the-token')
+        time.sleep(1) # this will delay the remaining reads on this response
+
+def handleResponse(req, interesting):
+    table.add(req)

src/ThreadedRequestEngine.kt

@@ -12,7 +12,7 @@ import javax.net.SocketFactory
 import javax.net.ssl.*
 import kotlin.concurrent.thread
 
-open class ThreadedRequestEngine(url: String, val threads: Int, maxQueueSize: Int, val readFreq: Int, val requestsPerConnection: Int, override val maxRetriesPerRequest: Int, override val callback: (Request, Boolean) -> Boolean, val timeout: Int, override var readCallback: ((String) -> Boolean)?): RequestEngine() {
+open class ThreadedRequestEngine(url: String, val threads: Int, maxQueueSize: Int, val readFreq: Int, val requestsPerConnection: Int, override val maxRetriesPerRequest: Int, override val callback: (Request, Boolean) -> Boolean, val timeout: Int, override var readCallback: ((String) -> Boolean)?, val readSize: Int): RequestEngine() {
 
     private val connectedLatch = CountDownLatch(threads)
 
@@ -108,6 +108,7 @@ open class ThreadedRequestEngine(url: String, val threads: Int, maxQueueSize: In
                 //(socket as SSLSocket).session.peerCertificates
                 socket!!.soTimeout = timeout * 1000
                 socket.tcpNoDelay = true
+                socket.receiveBufferSize = readSize
                 // todo tweak other TCP options for max performance
 
                 if(!connected) {
@@ -158,7 +159,7 @@ open class ThreadedRequestEngine(url: String, val threads: Int, maxQueueSize: In
 
                     }
 
-                    val readBuffer = ByteArray(1024)
+                    val readBuffer = ByteArray(readSize)
                     var buffer = ""
 
                     for (k in 1..readCount) {

src/fast-http.kt

@@ -31,7 +31,7 @@ class Engine:
 
 class RequestEngine:
 
-    def __init__(self, endpoint, callback=None, engine=Engine.THREADED, concurrentConnections=50, requestsPerConnection=100, pipeline=False, maxQueueSize=100, timeout=5, maxRetriesPerRequest=3, readCallback=None):
+    def __init__(self, endpoint, callback=None, engine=Engine.THREADED, concurrentConnections=50, requestsPerConnection=100, pipeline=False, maxQueueSize=100, timeout=5, maxRetriesPerRequest=3, readCallback=None, readSize = 1024):
         concurrentConnections = int(concurrentConnections)
         requestsPerConnection = int(requestsPerConnection)
 
@@ -50,9 +50,9 @@ class RequestEngine:
                 print('requestsPerConnection has been forced to 1 and pipelining has been disabled due to Burp engine limitations')
             if(readCallback != None):
                 print('Read callbacks are not supported in the Burp request engine. Try Engine.THREADED instead.')
-            self.engine = burp.BurpRequestEngine(endpoint, concurrentConnections, maxQueueSize, maxRetriesPerRequest, callback, readCallback)
+            self.engine = burp.BurpRequestEngine(endpoint, concurrentConnections, maxQueueSize, maxRetriesPerRequest, callback, readCallback, readSize)
         elif(engine == Engine.THREADED):
-            self.engine = burp.ThreadedRequestEngine(endpoint, concurrentConnections, maxQueueSize, readFreq, requestsPerConnection, maxRetriesPerRequest, callback, timeout, readCallback)
+            self.engine = burp.ThreadedRequestEngine(endpoint, concurrentConnections, maxQueueSize, readFreq, requestsPerConnection, maxRetriesPerRequest, callback, timeout, readCallback, readSize)
         elif(engine == Engine.ASYNC):
             self.engine = burp.AsyncRequestEngine(endpoint, concurrentConnections, readFreq, requestsPerConnection, False, callback)
         elif(engine == Engine.HTTP2):