feat: compress replay data (#1436)

src/__tests__/extensions/replay/sessionrecording.test.ts

@@ -89,11 +89,60 @@ const createIncrementalSnapshot = (event = {}): incrementalSnapshotEvent => ({
     ...event,
 })
 
-const createCustomSnapshot = (event = {}): customEvent => ({
+const createIncrementalMouseEvent = () => {
+    return createIncrementalSnapshot({
+        data: {
+            source: 2,
+            positions: [
+                {
+                    id: 1,
+                    x: 100,
+                    y: 200,
+                    timeOffset: 100,
+                },
+            ],
+        },
+    })
+}
+
+const createIncrementalMutationEvent = () => {
+    const mutationData = {
+        texts: [],
+        attributes: [],
+        removes: [],
+        adds: [],
+        isAttachIframe: true,
+    }
+    return createIncrementalSnapshot({
+        data: {
+            source: 0,
+            ...mutationData,
+        },
+    })
+}
+
+const createIncrementalStyleSheetEvent = () => {
+    return createIncrementalSnapshot({
+        data: {
+            // doesn't need to be a valid style sheet event
+            source: 8,
+            id: 1,
+            styleId: 1,
+            removes: [],
+            adds: [],
+            replace: 'something',
+            replaceSync: 'something',
+        },
+    })
+}
+
+const createCustomSnapshot = (event = {}, payload = {}): customEvent => ({
     type: EventType.Custom,
     data: {
         tag: 'custom',
-        payload: {},
+        payload: {
+            ...payload,
+        },
     },
     ...event,
 })
@@ -1937,4 +1986,164 @@ describe('SessionRecording', () => {
             expect((sessionRecording as any)['_tryAddCustomEvent']).not.toHaveBeenCalled()
         })
     })
+
+    describe('when compression is active', () => {
+        const captureOptions = {
+            _batchKey: 'recordings',
+            _noTruncate: true,
+            _url: 'https://test.com/s/',
+            skip_client_rate_limiting: true,
+        }
+
+        beforeEach(() => {
+            posthog.config.session_recording.compress_events = true
+            sessionRecording.afterDecideResponse(makeDecideResponse({ sessionRecording: { endpoint: '/s/' } }))
+            sessionRecording.startIfEnabledOrStop()
+        })
+
+        it('compresses full snapshot data', () => {
+            _emit(createFullSnapshot())
+            sessionRecording['_flushBuffer']()
+
+            expect(posthog.capture).toHaveBeenCalledWith(
+                '$snapshot',
+                {
+                    $snapshot_data: [
+                        {
+                            data: expect.any(String),
+                            cv: '2024-10',
+                            type: 2,
+                        },
+                    ],
+                    $session_id: sessionId,
+                    $snapshot_bytes: expect.any(Number),
+                    $window_id: 'windowId',
+                },
+                captureOptions
+            )
+        })
+
+        it('compresses incremental snapshot mutation data', () => {
+            _emit(createIncrementalMutationEvent())
+            sessionRecording['_flushBuffer']()
+
+            expect(posthog.capture).toHaveBeenCalledWith(
+                '$snapshot',
+                {
+                    $snapshot_data: [
+                        {
+                            cv: '2024-10',
+                            data: {
+                                adds: expect.any(String),
+                                texts: expect.any(String),
+                                removes: expect.any(String),
+                                attributes: expect.any(String),
+                                isAttachIframe: true,
+                                source: 0,
+                            },
+                            type: 3,
+                        },
+                    ],
+                    $session_id: sessionId,
+                    $snapshot_bytes: expect.any(Number),
+                    $window_id: 'windowId',
+                },
+                captureOptions
+            )
+        })
+
+        it('compresses incremental snapshot style data', () => {
+            _emit(createIncrementalStyleSheetEvent())
+            sessionRecording['_flushBuffer']()
+
+            expect(posthog.capture).toHaveBeenCalledWith(
+                '$snapshot',
+                {
+                    $snapshot_data: [
+                        {
+                            data: {
+                                adds: expect.any(String),
+                                id: 1,
+                                removes: expect.any(String),
+                                replace: 'something',
+                                replaceSync: 'something',
+                                source: 8,
+                                styleId: 1,
+                            },
+                            cv: '2024-10',
+                            type: 3,
+                        },
+                    ],
+                    $session_id: sessionId,
+                    $snapshot_bytes: expect.any(Number),
+                    $window_id: 'windowId',
+                },
+                captureOptions
+            )
+        })
+
+        it('does not compress incremental snapshot non full data', () => {
+            const mouseEvent = createIncrementalMouseEvent()
+            _emit(mouseEvent)
+            sessionRecording['_flushBuffer']()
+
+            expect(posthog.capture).toHaveBeenCalledWith(
+                '$snapshot',
+                {
+                    $snapshot_data: [mouseEvent],
+                    $session_id: sessionId,
+                    $snapshot_bytes: 86,
+                    $window_id: 'windowId',
+                },
+                captureOptions
+            )
+        })
+
+        it('does not compress custom events', () => {
+            _emit(createCustomSnapshot(undefined, { tag: 'wat' }))
+            sessionRecording['_flushBuffer']()
+
+            expect(posthog.capture).toHaveBeenCalledWith(
+                '$snapshot',
+                {
+                    $snapshot_data: [
+                        {
+                            data: {
+                                payload: { tag: 'wat' },
+                                tag: 'custom',
+                            },
+                            type: 5,
+                        },
+                    ],
+                    $session_id: sessionId,
+                    $snapshot_bytes: 58,
+                    $window_id: 'windowId',
+                },
+                captureOptions
+            )
+        })
+
+        it('does not compress meta events', () => {
+            _emit(createMetaSnapshot())
+            sessionRecording['_flushBuffer']()
+
+            expect(posthog.capture).toHaveBeenCalledWith(
+                '$snapshot',
+                {
+                    $snapshot_data: [
+                        {
+                            type: META_EVENT_TYPE,
+                            data: {
+                                href: 'https://has-to-be-present-or-invalid.com',
+                            },
+                        },
+                    ],
+                    $session_id: sessionId,
+                    $snapshot_bytes: 69,
+                    $window_id: 'windowId',
+                },
+                captureOptions
+            )
+        })
+    })
 })

src/extensions/replay/sessionrecording.ts

@@ -25,6 +25,7 @@ import { assignableWindow, document, window } from '../../utils/globals'
 import { buildNetworkRequestOptions } from './config'
 import { isLocalhost } from '../../utils/request-utils'
 import { MutationRateLimiter } from './mutation-rate-limiter'
+import { gzipSync, strFromU8, strToU8 } from 'fflate'
 
 const BASE_ENDPOINT = '/s/'
 
@@ -88,6 +89,95 @@ const newQueuedEvent = (rrwebMethod: () => void): QueuedRRWebEvent => ({
 
 const LOGGER_PREFIX = '[SessionRecording]'
 
+type compressedFullSnapshotEvent = {
+    type: EventType.FullSnapshot
+    data: string
+}
+
+type compressedIncrementalSnapshotEvent = {
+    type: EventType.IncrementalSnapshot
+    data: {
+        source: IncrementalSource
+        texts: string
+        attributes: string
+        removes: string
+        adds: string
+    }
+}
+
+type compressedIncrementalStyleSnapshotEvent = {
+    type: EventType.IncrementalSnapshot
+    data: {
+        source: IncrementalSource.StyleSheetRule
+        id?: number
+        styleId?: number
+        replace?: string
+        replaceSync?: string
+        adds: string
+        removes: string
+    }
+}
+
+export type compressedEvent =
+    | compressedIncrementalStyleSnapshotEvent
+    | compressedFullSnapshotEvent
+    | compressedIncrementalSnapshotEvent
+export type compressedEventWithTime = compressedEvent & {
+    timestamp: number
+    delay?: number
+    // marker for compression version
+    cv: '2024-10'
+}
+
+function gzipToString(data: unknown): string {
+    return strFromU8(gzipSync(strToU8(JSON.stringify(data))), true)
+}
+
+// rrweb's packer takes an event and returns a string or the reverse on unpact,
+// but we want to be able to inspect metadata during ingestion, and don't want to compress the entire event
+// so we have a custom packer that only compresses part of some events
+function compressEvent(event: eventWithTime, ph: PostHog): eventWithTime | compressedEventWithTime {
+    try {
+        if (event.type === EventType.FullSnapshot) {
+            return {
+                ...event,
+                data: gzipToString(event.data),
+                cv: '2024-10',
+            }
+        }
+        if (event.type === EventType.IncrementalSnapshot && event.data.source === IncrementalSource.Mutation) {
+            return {
+                ...event,
+                cv: '2024-10',
+                data: {
+                    ...event.data,
+                    texts: gzipToString(event.data.texts),
+                    attributes: gzipToString(event.data.attributes),
+                    removes: gzipToString(event.data.removes),
+                    adds: gzipToString(event.data.adds),
+                },
+            }
+        }
+        if (event.type === EventType.IncrementalSnapshot && event.data.source === IncrementalSource.StyleSheetRule) {
+            return {
+                ...event,
+                cv: '2024-10',
+                data: {
+                    ...event.data,
+                    adds: gzipToString(event.data.adds),
+                    removes: gzipToString(event.data.removes),
+                },
+            }
+        }
+    } catch (e: unknown) {
+        logger.error(LOGGER_PREFIX + ' could not compress event', e)
+        ph.captureException((e as Error) || 'e was not an error', {
+            attempted_event_type: event?.type || 'no event type',
+        })
+    }
+    return event
+}
+
 export class SessionRecording {
     private _endpoint: string
     private flushBufferTimer?: any
@@ -795,11 +885,10 @@ export class SessionRecording {
 
         // TODO: Re-add ensureMaxMessageSize once we are confident in it
         const event = truncateLargeConsoleLogs(throttledEvent)
-        const size = estimateSize(event)
 
         this._updateWindowAndSessionIds(event)
 
-        // When in an idle state we keep recording, but don't capture the events
+        // When in an idle state we keep recording, but don't capture the events,
         // but we allow custom events even when idle
         if (this.isIdle && event.type !== EventType.Custom) {
             return
@@ -817,9 +906,13 @@ export class SessionRecording {
             }
         }
 
+        const eventToSend = this.instance.config.session_recording.compress_events
+            ? compressEvent(event, this.instance)
+            : event
+        const size = estimateSize(eventToSend)
         const properties = {
             $snapshot_bytes: size,
-            $snapshot_data: event,
+            $snapshot_data: eventToSend,
             $session_id: this.sessionId,
             $window_id: this.windowId,
         }

src/types.ts

@@ -277,6 +277,8 @@ export interface SessionRecordingOptions {
     recordBody?: boolean
     // ADVANCED: while a user is active we take a full snapshot of the browser every interval. For very few sites playback performance might be better with different interval. Set to 0 to disable
     full_snapshot_interval_millis?: number
+    // PREVIEW: whether to compress part of the events before sending them to the server, this is a preview feature and may change without notice
+    compress_events?: boolean
 }
 
 export type SessionIdChangedCallback = (