Save to .pcap.gz instead of .pcap (#24)

Trades CPU for RAM and disk IO.  Both RAM and (especially) disk IO are
scarce resources on cloud machines, so this is a good trade.

demuxer/tcp_test.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"os/exec"
 	"sync"
 	"testing"
 	"time"
@@ -165,14 +166,15 @@ func TestTCPWithRealPcaps(t *testing.T) {
 	var err1, err2 error
 	err1 = errors.New("start out with an error")
 	for err1 != nil || err2 != nil {
-		_, err1 = os.Stat(dir + "/2013/10/31/flow1.pcap")
-		_, err2 = os.Stat(dir + "/2013/10/30/flow2.pcap")
+		_, err1 = os.Stat(dir + "/2013/10/31/flow1.pcap.gz")
+		_, err2 = os.Stat(dir + "/2013/10/30/flow2.pcap.gz")
 		time.Sleep(100 * time.Millisecond)
 	}
 
 	// Verify the files' contents.
+	rtx.Must(exec.Command("gunzip", dir+"/2013/10/31/flow1.pcap.gz").Run(), "Could not unzip flow1")
 	handle, err = pcap.OpenOffline(dir + "/2013/10/31/flow1.pcap")
-	rtx.Must(err, "Could not open golden pcap file")
+	rtx.Must(err, "Could not open golden pcap file: flow1.pcap")
 	ps = gopacket.NewPacketSource(handle, handle.LinkType())
 	v4 := make([]gopacket.Packet, 0)
 	for p := range ps.Packets() {
@@ -182,8 +184,9 @@ func TestTCPWithRealPcaps(t *testing.T) {
 		t.Errorf("%+v should have length 12 not %d", v4, len(v4))
 	}
 
+	rtx.Must(exec.Command("gunzip", dir+"/2013/10/30/flow2.pcap.gz").Run(), "Could not unzip flow2")
 	handle, err = pcap.OpenOffline(dir + "/2013/10/30/flow2.pcap")
-	rtx.Must(err, "Could not open golden pcap file")
+	rtx.Must(err, "Could not open golden pcap file: flow2.pcap")
 	ps = gopacket.NewPacketSource(handle, handle.LinkType())
 	v6 := make([]gopacket.Packet, 0)
 	for p := range ps.Packets() {

saver/tcp.go

@@ -3,6 +3,7 @@ package saver
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"io/ioutil"
 	"log"
@@ -63,7 +64,7 @@ type UUIDEvent struct {
 }
 
 func filename(dir string, e UUIDEvent) (string, string) {
-	return path.Join(dir, e.Timestamp.Format("2006/01/02")), e.UUID + ".pcap"
+	return path.Join(dir, e.Timestamp.Format("2006/01/02")), e.UUID + ".pcap.gz"
 }
 
 type statusSetter interface {
@@ -141,9 +142,10 @@ func (t *TCP) savePackets(ctx context.Context, duration time.Duration) {
 	defer derivedCancel()
 
 	buff := &bytes.Buffer{}
+	zip := gzip.NewWriter(buff)
 
 	// Write PCAP data to the buffer.
-	w := pcapgo.NewWriterNanos(buff)
+	w := pcapgo.NewWriterNanos(zip)
 	// Now save packets until the stream is done or the context is canceled.
 	t.state.Set("readingpackets")
 	// Read the first packet to determine the TCP+IP header size (as IPv6 is variable in size)
@@ -187,6 +189,8 @@ func (t *TCP) savePackets(ctx context.Context, duration time.Duration) {
 		}
 		t.savePacket(w, p, headerLen)
 	}
+	zip.Close()
+	// buff now contains a complete .gz file, complete with footer.
 
 	// Read the UUID to determine the filename
 	t.state.Set("uuidwait")

saver/tcp_test.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"os/exec"
 	"reflect"
 	"testing"
 	"time"
@@ -17,6 +18,10 @@ import (
 	"github.com/m-lab/go/rtx"
 )
 
+func init() {
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+}
+
 func TestMinInt(t *testing.T) {
 	for _, c := range []struct{ x, y, want int }{
 		{1, 0, 0},
@@ -276,7 +281,9 @@ func TestSaverWithRealv4Data(t *testing.T) {
 		time.Sleep(time.Millisecond)
 	}
 
-	log.Println("reading data from", dir+"/2000/01/02/testUUID.pcap")
+	log.Println("reading data from", dir+"/2000/01/02/testUUID.pcap.gz")
+	rtx.Must(exec.Command("gunzip", dir+"/2000/01/02/testUUID.pcap.gz").Run(), "Could not unzip")
+
 	handle, err := pcap.OpenOffline(dir + "/2000/01/02/testUUID.pcap")
 	rtx.Must(err, "Could not open written pcap file")
 	ps := gopacket.NewPacketSource(handle, handle.LinkType())
@@ -352,6 +359,7 @@ func TestSaverWithRealv6Data(t *testing.T) {
 	}
 
 	log.Println("reading data from", dir+"/2000/01/02/testUUID.pcap")
+	rtx.Must(exec.Command("gunzip", dir+"/2000/01/02/testUUID.pcap.gz").Run(), "Could not unzip")
 	handle, err := pcap.OpenOffline(dir + "/2000/01/02/testUUID.pcap")
 	rtx.Must(err, "Could not open written pcap file")
 	ps := gopacket.NewPacketSource(handle, handle.LinkType())