feat: default replay partial compression on (#1445)

src/__tests__/extensions/replay/sessionrecording.test.ts

@@ -105,9 +105,9 @@ const createIncrementalMouseEvent = () => {
     })
 }
 
-const createIncrementalMutationEvent = () => {
+const createIncrementalMutationEvent = (mutations?: { texts: any[] }) => {
     const mutationData = {
-        texts: [],
+        texts: mutations?.texts || [],
         attributes: [],
         removes: [],
         adds: [],
@@ -121,15 +121,15 @@ const createIncrementalMutationEvent = () => {
     })
 }
 
-const createIncrementalStyleSheetEvent = () => {
+const createIncrementalStyleSheetEvent = (mutations?: { adds: any[] }) => {
     return createIncrementalSnapshot({
         data: {
             // doesn't need to be a valid style sheet event
             source: 8,
             id: 1,
             styleId: 1,
             removes: [],
-            adds: [],
+            adds: mutations.adds || [],
             replace: 'something',
             replaceSync: 'something',
         },
@@ -1009,6 +1009,7 @@ describe('SessionRecording', () => {
         })
 
         it('can emit when there are circular references', () => {
+            posthog.config.session_recording.compress_events = false
             sessionRecording.afterDecideResponse(makeDecideResponse({ sessionRecording: { endpoint: '/s/' } }))
             sessionRecording.startIfEnabledOrStop()
 
@@ -1271,6 +1272,10 @@ describe('SessionRecording', () => {
                 type: INCREMENTAL_SNAPSHOT_EVENT_TYPE,
                 data: {
                     source: 0,
+                    adds: [],
+                    attributes: [],
+                    removes: [],
+                    texts: [],
                 },
                 timestamp: activityTimestamp,
             }
@@ -1490,7 +1495,7 @@ describe('SessionRecording', () => {
             expect(sessionRecording['buffer']).toEqual({
                 data: [firstSnapshotEvent, secondSnapshot],
                 sessionId: firstSessionId,
-                size: 136,
+                size: 186,
                 windowId: expect.any(String),
             })
 
@@ -1514,7 +1519,7 @@ describe('SessionRecording', () => {
                 {
                     $snapshot_data: [firstSnapshotEvent, secondSnapshot],
                     $session_id: firstSessionId,
-                    $snapshot_bytes: 136,
+                    $snapshot_bytes: 186,
                     $window_id: expect.any(String),
                 },
                 {
@@ -1576,7 +1581,7 @@ describe('SessionRecording', () => {
             expect(sessionRecording['buffer']).toEqual({
                 data: [firstSnapshotEvent, secondSnapshot],
                 sessionId: firstSessionId,
-                size: 136,
+                size: 186,
                 windowId: expect.any(String),
             })
 
@@ -1604,7 +1609,7 @@ describe('SessionRecording', () => {
                 {
                     $snapshot_data: [firstSnapshotEvent, secondSnapshot],
                     $session_id: firstSessionId,
-                    $snapshot_bytes: 136,
+                    $snapshot_bytes: 186,
                     $window_id: expect.any(String),
                 },
                 {
@@ -1630,7 +1635,7 @@ describe('SessionRecording', () => {
                 {
                     $snapshot_data: [firstSnapshotEvent, secondSnapshot],
                     $session_id: firstSessionId,
-                    $snapshot_bytes: 136,
+                    $snapshot_bytes: 186,
                     $window_id: expect.any(String),
                 },
                 {
@@ -1969,7 +1974,13 @@ describe('SessionRecording', () => {
         })
 
         it('compresses full snapshot data', () => {
-            _emit(createFullSnapshot())
+            _emit(
+                createFullSnapshot({
+                    data: {
+                        content: Array(30).fill(uuidv7()).join(''),
+                    },
+                })
+            )
             sessionRecording['_flushBuffer']()
 
             expect(posthog.capture).toHaveBeenCalledWith(
@@ -1990,8 +2001,29 @@ describe('SessionRecording', () => {
             )
         })
 
+        it('does not compress small full snapshot data', () => {
+            _emit(createFullSnapshot({ data: { content: 'small' } }))
+            sessionRecording['_flushBuffer']()
+
+            expect(posthog.capture).toHaveBeenCalledWith(
+                '$snapshot',
+                {
+                    $snapshot_data: [
+                        {
+                            data: { content: 'small' },
+                            type: 2,
+                        },
+                    ],
+                    $session_id: sessionId,
+                    $snapshot_bytes: expect.any(Number),
+                    $window_id: 'windowId',
+                },
+                captureOptions
+            )
+        })
+
         it('compresses incremental snapshot mutation data', () => {
-            _emit(createIncrementalMutationEvent())
+            _emit(createIncrementalMutationEvent({ texts: [Array(30).fill(uuidv7()).join('')] }))
             sessionRecording['_flushBuffer']()
 
             expect(posthog.capture).toHaveBeenCalledWith(
@@ -2020,7 +2052,7 @@ describe('SessionRecording', () => {
         })
 
         it('compresses incremental snapshot style data', () => {
-            _emit(createIncrementalStyleSheetEvent())
+            _emit(createIncrementalStyleSheetEvent({ adds: [Array(30).fill(uuidv7()).join('')] }))
             sessionRecording['_flushBuffer']()
 
             expect(posthog.capture).toHaveBeenCalledWith(
@@ -2049,6 +2081,8 @@ describe('SessionRecording', () => {
             )
         })
 
+        it('does not compress small incremental snapshot data', () => {})
+
         it('does not compress incremental snapshot non full data', () => {
             const mouseEvent = createIncrementalMouseEvent()
             _emit(mouseEvent)

src/extensions/replay/sessionrecording.ts

@@ -39,7 +39,9 @@ const BASE_ENDPOINT = '/s/'
 const FIVE_MINUTES = 1000 * 60 * 5
 const TWO_SECONDS = 2000
 export const RECORDING_IDLE_THRESHOLD_MS = FIVE_MINUTES
-export const RECORDING_MAX_EVENT_SIZE = 1024 * 1024 * 0.9 // ~1mb (with some wiggle room)
+const ONE_KB = 1024
+const PARTIAL_COMPRESSION_THRESHOLD = ONE_KB
+export const RECORDING_MAX_EVENT_SIZE = ONE_KB * ONE_KB * 0.9 // ~1mb (with some wiggle room)
 export const RECORDING_BUFFER_TIMEOUT = 2000 // 2 seconds
 export const SESSION_RECORDING_BATCH_KEY = 'recordings'
 
@@ -144,6 +146,11 @@ function gzipToString(data: unknown): string {
 // but we want to be able to inspect metadata during ingestion, and don't want to compress the entire event
 // so we have a custom packer that only compresses part of some events
 function compressEvent(event: eventWithTime, ph: PostHog): eventWithTime | compressedEventWithTime {
+    const originalSize = estimateSize(event)
+    if (originalSize < PARTIAL_COMPRESSION_THRESHOLD) {
+        return event
+    }
+
     try {
         if (event.type === EventType.FullSnapshot) {
             return {
@@ -937,10 +944,10 @@ export class SessionRecording {
             }
         }
 
-        const eventToSend = this.instance.config.session_recording.compress_events
-            ? compressEvent(event, this.instance)
-            : event
+        const eventToSend =
+            this.instance.config.session_recording.compress_events ?? true ? compressEvent(event, this.instance) : event
         const size = estimateSize(eventToSend)
+
         const properties = {
             $snapshot_bytes: size,
             $snapshot_data: eventToSend,

src/types.ts

@@ -277,7 +277,11 @@ export interface SessionRecordingOptions {
     recordBody?: boolean
     // ADVANCED: while a user is active we take a full snapshot of the browser every interval. For very few sites playback performance might be better with different interval. Set to 0 to disable
     full_snapshot_interval_millis?: number
-    // PREVIEW: whether to compress part of the events before sending them to the server, this is a preview feature and may change without notice
+    /*
+     ADVANCED: whether to partially compress rrweb events before sending them to the server,
+     defaults to true, can be set to false to disable partial compression
+     NB requests are still compressed when sent to the server regardless of this setting
+    */
     compress_events?: boolean
     /*
      ADVANCED: alters the threshold before a recording considers a user has become idle.