feat(web experiments): Emit web_experiment_applied event and do not r…

…ender experiments for bots (#1443)

1. Replace references to className with css which assigns the value of this field to the style attribute of an HTML element.
2. Introduce the ability to preview an Experiment variant in the browser without deploying the experiment.
3. Do not render web experiments if the viewer is a bot.
4. Emit the $web_experiment_applied event so we know if the visual changes actually got applied to a page.

playground/nextjs/pages/index.tsx

@@ -27,10 +27,14 @@ export default function Home() {
         <>
             <p className="italic my-2 text-gray-500">The current time is {time}</p>
 
-            <h2>Trigger posthog events</h2>
+            <h2>
+                Trigger posthog <span>events </span>
+            </h2>
             <div className="flex items-center gap-2 flex-wrap">
                 <button onClick={() => posthog.capture('Clicked button')}>Capture event</button>
-                <button onClick={() => posthog.capture('user_subscribed')}>Subscribe to newsletter</button>
+                <button id="subscribe-user-to-newsletter" onClick={() => posthog.capture('user_subscribed')}>
+                    Subscribe to newsletter
+                </button>
                 <button onClick={() => posthog.capture('user_unsubscribed')}>Unsubscribe from newsletter</button>
                 <button data-attr="autocapture-button">Autocapture buttons</button>
                 <a className="Button" data-attr="autocapture-button" href="#">

playground/nextjs/src/posthog.ts

@@ -50,6 +50,7 @@ if (typeof window !== 'undefined') {
             recordCrossOriginIframes: true,
         },
         debug: true,
+        disable_web_experiments: false,
         scroll_root_selector: ['#scroll_element', 'html'],
         persistence: cookieConsentGiven() ? 'localStorage+cookie' : 'memory',
         person_profiles: PERSON_PROCESSING_MODE === 'never' ? 'identified_only' : PERSON_PROCESSING_MODE,

src/__tests__/web-experiments.test.ts

@@ -38,7 +38,7 @@ describe('Web Experimentation', () => {
                 transforms: [
                     {
                         selector: '#set-user-properties',
-                        className: 'primary',
+                        css: 'font-size:40px',
                     },
                 ],
             },
@@ -89,7 +89,7 @@ describe('Web Experimentation', () => {
                     },
                 ],
             },
-            control: {
+            icontains: {
                 conditions: { url: 'checkout', urlMatchType: 'icontains' },
                 transforms: [
                     {
@@ -99,6 +99,15 @@ describe('Web Experimentation', () => {
                     },
                 ],
             },
+            control: {
+                transforms: [
+                    {
+                        selector: '#set-user-properties',
+                        text: 'Sign up',
+                        html: 'Sign up',
+                    },
+                ],
+            },
         },
     } as unknown as WebExperiment
 
@@ -114,6 +123,7 @@ describe('Web Experimentation', () => {
             } as unknown as PostHogConfig,
             persistence: persistence,
             get_property: jest.fn(),
+            capture: jest.fn(),
             _send_request: jest
                 .fn()
                 .mockImplementation(({ callback }) => callback({ statusCode: 200, json: experimentsResponse })),
@@ -130,8 +140,8 @@ describe('Web Experimentation', () => {
         elTarget.id = 'primary_button'
         // eslint-disable-next-line no-restricted-globals
         const elParent = document.createElement('span')
-        elParent.innerText = 'unassigned'
-        elParent.className = 'unassigned'
+        elParent.innerText = 'original'
+        elParent.className = 'original'
         elParent.appendChild(elTarget)
         // eslint-disable-next-line no-restricted-globals
         document.querySelectorAll = function () {
@@ -167,8 +177,8 @@ describe('Web Experimentation', () => {
         } as unknown as DecideResponse)
 
         switch (expectedProperty) {
-            case 'className':
-                expect(elParent.className).toEqual(value)
+            case 'css':
+                expect(elParent.getAttribute('style')).toEqual(value)
                 break
 
             case 'innerText':
@@ -181,6 +191,24 @@ describe('Web Experimentation', () => {
         }
     }
 
+    describe('bot detection', () => {
+        it('does not apply web experiment if viewer is a bot', () => {
+            experimentsResponse = {
+                experiments: [buttonWebExperimentWithUrlConditions],
+            }
+            const webExperiment = new WebExperiments(posthog)
+            webExperiment._is_bot = () => true
+            const elParent = createTestDocument()
+
+            webExperiment.afterDecideResponse({
+                featureFlags: {
+                    'signup-button-test': 'Sign me up',
+                },
+            } as unknown as DecideResponse)
+            expect(elParent.innerText).toEqual('original')
+        })
+    })
+
     describe('url match conditions', () => {
         it('exact location match', () => {
             const testLocation = 'https://example.com/Signup'
@@ -211,7 +239,7 @@ describe('Web Experimentation', () => {
                 },
             }
             const testLocation = 'https://example.com/landing-page?utm_campaign=marketing&utm_medium=mobile'
-            const expectedText = 'unassigned'
+            const expectedText = 'original'
             testUrlMatch(testLocation, expectedText)
         })
     })
@@ -238,26 +266,75 @@ describe('Web Experimentation', () => {
 
             posthog.requestRouter = new RequestRouter(disabledPostHog)
             webExperiment = new WebExperiments(disabledPostHog)
-            assertElementChanged('control', 'innerText', 'unassigned')
+            assertElementChanged('control', 'innerText', 'original')
         })
 
         it('can set text of Span Element', async () => {
             experimentsResponse = {
                 experiments: [signupButtonWebExperimentWithFeatureFlag],
             }
 
-            assertElementChanged('control', 'innerText', 'Sign up')
+            assertElementChanged('Signup', 'innerText', 'Sign me up')
+            expect(posthog.capture).toHaveBeenCalledWith('$web_experiment_applied', {
+                $web_experiment_document_url:
+                    'https://example.com/landing-page?utm_campaign=marketing&utm_medium=mobile',
+                $web_experiment_elements_modified: 1,
+                $web_experiment_name: 'Signup button test',
+                $web_experiment_variant: 'Signup',
+            })
+        })
+
+        it('makes no modifications if control variant', () => {
+            experimentsResponse = {
+                experiments: [signupButtonWebExperimentWithFeatureFlag],
+            }
+            assertElementChanged('control', 'innerText', 'original')
+            expect(posthog.capture).toHaveBeenCalledWith('$web_experiment_applied', {
+                $web_experiment_document_url:
+                    'https://example.com/landing-page?utm_campaign=marketing&utm_medium=mobile',
+                $web_experiment_elements_modified: 0,
+                $web_experiment_name: 'Signup button test',
+                $web_experiment_variant: 'control',
+            })
+        })
+
+        it('can render previews based on URL params', () => {
+            experimentsResponse = {
+                experiments: [buttonWebExperimentWithUrlConditions],
+            }
+            const webExperiment = new WebExperiments(posthog)
+            const elParent = createTestDocument()
+            const original = WebExperiments.getWindowLocation
+            WebExperiments.getWindowLocation = () => {
+                // eslint-disable-next-line compat/compat
+                return new URL(
+                    'https://example.com/landing-page?__experiment_id=3&__experiment_variant=Signup'
+                ) as unknown as Location
+            }
+
+            webExperiment.previewWebExperiment()
+
+            WebExperiments.getWindowLocation = original
+            expect(elParent.innerText).toEqual('Sign me up')
+            expect(posthog.capture).toHaveBeenCalledWith('$web_experiment_applied', {
+                $web_experiment_document_url:
+                    'https://example.com/landing-page?__experiment_id=3&__experiment_variant=Signup',
+                $web_experiment_elements_modified: 1,
+                $web_experiment_name: 'Signup button test',
+                $web_experiment_variant: 'Signup',
+                $web_experiment_preview: true,
+            })
         })
 
-        it('can set className of Span Element', async () => {
+        it('can set css of Span Element', async () => {
             experimentsResponse = {
                 experiments: [signupButtonWebExperimentWithFeatureFlag],
             }
 
-            assertElementChanged('css-transform', 'className', 'primary')
+            assertElementChanged('css-transform', 'css', 'font-size:40px')
         })
 
-        it('can set innerHtml of Span Element', async () => {
+        it('can set innerHTML of Span Element', async () => {
             experimentsResponse = {
                 experiments: [signupButtonWebExperimentWithFeatureFlag],
             }

src/web-experiments-types.ts

@@ -8,7 +8,7 @@ export interface WebExperimentTransform {
     text?: string
     html?: string
     imgUrl?: string
-    className?: string
+    css?: string
 }
 
 export type WebExperimentUrlMatchType = 'regex' | 'not_regex' | 'exact' | 'is_not' | 'icontains' | 'not_icontains'