From ba1c07fe6ccf03555ad4ea9decd9e77e961479c7 Mon Sep 17 00:00:00 2001 From: Phani Raj Date: Tue, 8 Oct 2024 04:33:20 +0200 Subject: [PATCH] feat(web experiments): Emit web_experiment_applied event and do not render experiments for bots (#1443) 1. Replace references to className with css which assigns the value of this field to the style attribute of an HTML element. 2. Introduce the ability to preview an Experiment variant in the browser without deploying the experiment. 3. Do not render web experiments if the viewer is a bot. 4. Emit the $web_experiment_applied event so we know if the visual changes actually got applied to a page. --- playground/nextjs/pages/index.tsx | 8 +- playground/nextjs/src/posthog.ts | 1 + src/__tests__/web-experiments.test.ts | 101 +++++++++++++++++++--- src/web-experiments-types.ts | 2 +- src/web-experiments.ts | 115 +++++++++++++++++++++++--- 5 files changed, 199 insertions(+), 28 deletions(-) diff --git a/playground/nextjs/pages/index.tsx b/playground/nextjs/pages/index.tsx index 0a893e106..3c3a06cbb 100644 --- a/playground/nextjs/pages/index.tsx +++ b/playground/nextjs/pages/index.tsx @@ -27,10 +27,14 @@ export default function Home() { <>

The current time is {time}

-

Trigger posthog events

+

+ Trigger posthog events +

- + diff --git a/playground/nextjs/src/posthog.ts b/playground/nextjs/src/posthog.ts index ae913c0aa..6a4062a34 100644 --- a/playground/nextjs/src/posthog.ts +++ b/playground/nextjs/src/posthog.ts @@ -50,6 +50,7 @@ if (typeof window !== 'undefined') { recordCrossOriginIframes: true, }, debug: true, + disable_web_experiments: false, scroll_root_selector: ['#scroll_element', 'html'], persistence: cookieConsentGiven() ? 'localStorage+cookie' : 'memory', person_profiles: PERSON_PROCESSING_MODE === 'never' ? 'identified_only' : PERSON_PROCESSING_MODE, diff --git a/src/__tests__/web-experiments.test.ts b/src/__tests__/web-experiments.test.ts index cbdc8a558..8e9ace341 100644 --- a/src/__tests__/web-experiments.test.ts +++ b/src/__tests__/web-experiments.test.ts @@ -38,7 +38,7 @@ describe('Web Experimentation', () => { transforms: [ { selector: '#set-user-properties', - className: 'primary', + css: 'font-size:40px', }, ], }, @@ -89,7 +89,7 @@ describe('Web Experimentation', () => { }, ], }, - control: { + icontains: { conditions: { url: 'checkout', urlMatchType: 'icontains' }, transforms: [ { @@ -99,6 +99,15 @@ describe('Web Experimentation', () => { }, ], }, + control: { + transforms: [ + { + selector: '#set-user-properties', + text: 'Sign up', + html: 'Sign up', + }, + ], + }, }, } as unknown as WebExperiment @@ -114,6 +123,7 @@ describe('Web Experimentation', () => { } as unknown as PostHogConfig, persistence: persistence, get_property: jest.fn(), + capture: jest.fn(), _send_request: jest .fn() .mockImplementation(({ callback }) => callback({ statusCode: 200, json: experimentsResponse })), @@ -130,8 +140,8 @@ describe('Web Experimentation', () => { elTarget.id = 'primary_button' // eslint-disable-next-line no-restricted-globals const elParent = document.createElement('span') - elParent.innerText = 'unassigned' - elParent.className = 'unassigned' + elParent.innerText = 'original' + elParent.className = 'original' elParent.appendChild(elTarget) // eslint-disable-next-line no-restricted-globals document.querySelectorAll = function () { @@ -167,8 +177,8 @@ describe('Web Experimentation', () => { } as unknown as DecideResponse) switch (expectedProperty) { - case 'className': - expect(elParent.className).toEqual(value) + case 'css': + expect(elParent.getAttribute('style')).toEqual(value) break case 'innerText': @@ -181,6 +191,24 @@ describe('Web Experimentation', () => { } } + describe('bot detection', () => { + it('does not apply web experiment if viewer is a bot', () => { + experimentsResponse = { + experiments: [buttonWebExperimentWithUrlConditions], + } + const webExperiment = new WebExperiments(posthog) + webExperiment._is_bot = () => true + const elParent = createTestDocument() + + webExperiment.afterDecideResponse({ + featureFlags: { + 'signup-button-test': 'Sign me up', + }, + } as unknown as DecideResponse) + expect(elParent.innerText).toEqual('original') + }) + }) + describe('url match conditions', () => { it('exact location match', () => { const testLocation = 'https://example.com/Signup' @@ -211,7 +239,7 @@ describe('Web Experimentation', () => { }, } const testLocation = 'https://example.com/landing-page?utm_campaign=marketing&utm_medium=mobile' - const expectedText = 'unassigned' + const expectedText = 'original' testUrlMatch(testLocation, expectedText) }) }) @@ -238,7 +266,7 @@ describe('Web Experimentation', () => { posthog.requestRouter = new RequestRouter(disabledPostHog) webExperiment = new WebExperiments(disabledPostHog) - assertElementChanged('control', 'innerText', 'unassigned') + assertElementChanged('control', 'innerText', 'original') }) it('can set text of Span Element', async () => { @@ -246,18 +274,67 @@ describe('Web Experimentation', () => { experiments: [signupButtonWebExperimentWithFeatureFlag], } - assertElementChanged('control', 'innerText', 'Sign up') + assertElementChanged('Signup', 'innerText', 'Sign me up') + expect(posthog.capture).toHaveBeenCalledWith('$web_experiment_applied', { + $web_experiment_document_url: + 'https://example.com/landing-page?utm_campaign=marketing&utm_medium=mobile', + $web_experiment_elements_modified: 1, + $web_experiment_name: 'Signup button test', + $web_experiment_variant: 'Signup', + }) + }) + + it('makes no modifications if control variant', () => { + experimentsResponse = { + experiments: [signupButtonWebExperimentWithFeatureFlag], + } + assertElementChanged('control', 'innerText', 'original') + expect(posthog.capture).toHaveBeenCalledWith('$web_experiment_applied', { + $web_experiment_document_url: + 'https://example.com/landing-page?utm_campaign=marketing&utm_medium=mobile', + $web_experiment_elements_modified: 0, + $web_experiment_name: 'Signup button test', + $web_experiment_variant: 'control', + }) + }) + + it('can render previews based on URL params', () => { + experimentsResponse = { + experiments: [buttonWebExperimentWithUrlConditions], + } + const webExperiment = new WebExperiments(posthog) + const elParent = createTestDocument() + const original = WebExperiments.getWindowLocation + WebExperiments.getWindowLocation = () => { + // eslint-disable-next-line compat/compat + return new URL( + 'https://example.com/landing-page?__experiment_id=3&__experiment_variant=Signup' + ) as unknown as Location + } + + webExperiment.previewWebExperiment() + + WebExperiments.getWindowLocation = original + expect(elParent.innerText).toEqual('Sign me up') + expect(posthog.capture).toHaveBeenCalledWith('$web_experiment_applied', { + $web_experiment_document_url: + 'https://example.com/landing-page?__experiment_id=3&__experiment_variant=Signup', + $web_experiment_elements_modified: 1, + $web_experiment_name: 'Signup button test', + $web_experiment_variant: 'Signup', + $web_experiment_preview: true, + }) }) - it('can set className of Span Element', async () => { + it('can set css of Span Element', async () => { experimentsResponse = { experiments: [signupButtonWebExperimentWithFeatureFlag], } - assertElementChanged('css-transform', 'className', 'primary') + assertElementChanged('css-transform', 'css', 'font-size:40px') }) - it('can set innerHtml of Span Element', async () => { + it('can set innerHTML of Span Element', async () => { experimentsResponse = { experiments: [signupButtonWebExperimentWithFeatureFlag], } diff --git a/src/web-experiments-types.ts b/src/web-experiments-types.ts index 25175d809..abc27339c 100644 --- a/src/web-experiments-types.ts +++ b/src/web-experiments-types.ts @@ -8,7 +8,7 @@ export interface WebExperimentTransform { text?: string html?: string imgUrl?: string - className?: string + css?: string } export type WebExperimentUrlMatchType = 'regex' | 'not_regex' | 'exact' | 'is_not' | 'icontains' | 'not_icontains' diff --git a/src/web-experiments.ts b/src/web-experiments.ts index 2499c5450..20b4bfd53 100644 --- a/src/web-experiments.ts +++ b/src/web-experiments.ts @@ -1,6 +1,6 @@ import { PostHog } from './posthog-core' import { DecideResponse } from './types' -import { window } from './utils/globals' +import { navigator, window } from './utils/globals' import { WebExperiment, WebExperimentsCallback, @@ -10,9 +10,10 @@ import { } from './web-experiments-types' import { WEB_EXPERIMENTS } from './constants' import { isNullish } from './utils/type-utils' -import { isUrlMatchingRegex } from './utils/request-utils' +import { getQueryParam, isUrlMatchingRegex } from './utils/request-utils' import { logger } from './utils/logger' import { Info } from './utils/event-utils' +import { isLikelyBot } from './utils/blocked-uas' export const webExperimentUrlValidationMap: Record< WebExperimentUrlMatchType, @@ -46,17 +47,17 @@ export class WebExperiments { } applyFeatureFlagChanges(flags: string[]) { - WebExperiments.logInfo('applying feature flags', flags) if (isNullish(this._flagToExperiments) || this.instance.config.disable_web_experiments) { return } + WebExperiments.logInfo('applying feature flags', flags) flags.forEach((flag) => { if (this._flagToExperiments && this._flagToExperiments?.has(flag)) { const selectedVariant = this.instance.getFeatureFlag(flag) as unknown as string const webExperiment = this._flagToExperiments?.get(flag) if (selectedVariant && webExperiment?.variants[selectedVariant]) { - WebExperiments.applyTransforms( + this.applyTransforms( webExperiment.name, selectedVariant, webExperiment.variants[selectedVariant].transforms @@ -67,9 +68,32 @@ export class WebExperiments { } afterDecideResponse(response: DecideResponse) { - this._featureFlags = response.featureFlags + if (this._is_bot()) { + WebExperiments.logInfo('Refusing to render web experiment since the viewer is a likely bot') + return + } + this._featureFlags = response.featureFlags this.loadIfEnabled() + this.previewWebExperiment() + } + + previewWebExperiment() { + const location = WebExperiments.getWindowLocation() + if (location?.search) { + const experimentID = getQueryParam(location?.search, '__experiment_id') + const variant = getQueryParam(location?.search, '__experiment_variant') + if (experimentID && variant) { + WebExperiments.logInfo(`previewing web experiments ${experimentID} && ${variant}`) + this.getWebExperiments( + (webExperiments) => { + this.showPreviewWebExperiment(parseInt(experimentID), variant, webExperiments) + }, + false, + true + ) + } + } } loadIfEnabled() { @@ -84,6 +108,7 @@ export class WebExperiments { this.getWebExperiments((webExperiments) => { WebExperiments.logInfo(`retrieved web experiments from the server`) this._flagToExperiments = new Map() + webExperiments.forEach((webExperiment) => { if ( webExperiment.feature_flag_key && @@ -102,7 +127,7 @@ export class WebExperiments { const selectedVariant = this._featureFlags[webExperiment.feature_flag_key] as unknown as string if (selectedVariant && webExperiment.variants[selectedVariant]) { - WebExperiments.applyTransforms( + this.applyTransforms( webExperiment.name, selectedVariant, webExperiment.variants[selectedVariant].transforms @@ -113,7 +138,7 @@ export class WebExperiments { const testVariant = webExperiment.variants[variant] const matchTest = WebExperiments.matchesTestVariant(testVariant) if (matchTest) { - WebExperiments.applyTransforms(webExperiment.name, variant, testVariant.transforms) + this.applyTransforms(webExperiment.name, variant, testVariant.transforms) } } } @@ -121,8 +146,8 @@ export class WebExperiments { }, forceReload) } - public getWebExperiments(callback: WebExperimentsCallback, forceReload: boolean) { - if (this.instance.config.disable_web_experiments) { + public getWebExperiments(callback: WebExperimentsCallback, forceReload: boolean, previewing?: boolean) { + if (this.instance.config.disable_web_experiments && !previewing) { return callback([]) } @@ -148,6 +173,20 @@ export class WebExperiments { }) } + private showPreviewWebExperiment(experimentID: number, variant: string, webExperiments: WebExperiment[]) { + const previewExperiments = webExperiments.filter((exp) => exp.id === experimentID) + if (previewExperiments && previewExperiments.length > 0) { + WebExperiments.logInfo( + `Previewing web experiment [${previewExperiments[0].name}] with variant [${variant}]` + ) + this.applyTransforms( + previewExperiments[0].name, + variant, + previewExperiments[0].variants[variant].transforms, + true + ) + } + } private static matchesTestVariant(testVariant: WebExperimentVariant) { if (isNullish(testVariant.conditions)) { return false @@ -211,17 +250,45 @@ export class WebExperiments { logger.info(`[WebExperiments] ${msg}`, args) } - private static applyTransforms(experiment: string, variant: string, transforms: WebExperimentTransform[]) { + private applyTransforms( + experiment: string, + variant: string, + transforms: WebExperimentTransform[], + isPreview?: boolean + ) { + if (this._is_bot()) { + WebExperiments.logInfo('Refusing to render web experiment since the viewer is a likely bot') + return + } + + if (variant === 'control') { + WebExperiments.logInfo('Control variants leave the page unmodified.') + if (this.instance && this.instance.capture) { + this.instance.capture('$web_experiment_applied', { + $web_experiment_name: experiment, + $web_experiment_preview: isPreview, + $web_experiment_variant: variant, + $web_experiment_document_url: WebExperiments.getWindowLocation()?.href, + $web_experiment_elements_modified: 0, + }) + } + + return + } + transforms.forEach((transform) => { if (transform.selector) { WebExperiments.logInfo( `applying transform of variant ${variant} for experiment ${experiment} `, transform ) + + let elementsModified = 0 // eslint-disable-next-line no-restricted-globals const elements = document?.querySelectorAll(transform.selector) elements?.forEach((element) => { const htmlElement = element as HTMLElement + elementsModified += 1 if (transform.attributes) { transform.attributes.forEach((attribute) => { switch (attribute.name) { @@ -248,14 +315,36 @@ export class WebExperiments { } if (transform.html) { - htmlElement.innerHTML = transform.html + if (htmlElement.parentElement) { + htmlElement.parentElement.innerHTML = transform.html + } else { + htmlElement.innerHTML = transform.html + } } - if (transform.className) { - htmlElement.className = transform.className + if (transform.css) { + htmlElement.setAttribute('style', transform.css) } }) + + if (this.instance && this.instance.capture) { + this.instance.capture('$web_experiment_applied', { + $web_experiment_name: experiment, + $web_experiment_variant: variant, + $web_experiment_preview: isPreview, + $web_experiment_document_url: WebExperiments.getWindowLocation()?.href, + $web_experiment_elements_modified: elementsModified, + }) + } } }) } + + _is_bot(): boolean | undefined { + if (navigator && this.instance) { + return isLikelyBot(navigator, this.instance.config.custom_blocked_useragents) + } else { + return undefined + } + } }