feat(e2e): support filtering references for retainer traces (#106)

Summary:
This diff adds a new callback that can be used to define a logic to decide whether a reference should be considered as part of the retainer trace. The callback is called for every reference (edge) in the heap snapshot.

The callback accepts the following parameters
  * `edge` - the reference (edge) that is considered for calcualting the retainer trace
  * `snapshot` - the snapshot of target interaction
  * `isReferenceUsedByDefault` - MemLab has its own default logic for whether a reference should be considered as part of the retainer trace, if this parameter is true, it means MemLab will consider this reference when calculating the retainer trace.

The return value indicates whether the given reference should be considered when calculating the retainer trace. Note that when this callback returns true, the reference will only be considered as a candidate for retainer trace, so it may or may not be included in the retainer trace; however, if this callback returns false, the reference will be excluded.

Note that by excluding a dominator reference of an object (i.e., an edge that must be traveled through to reach the heap object from GC roots), the object will be considered as unreachable in the heap graph; and therefore, the reference and heap object will not be included in the retainer trace detection and retainer size calculation.

```
// save as leak-filter.js
module.exports = {
  retainerReferenceFilter(edge, _snapshot, _leakedNodeIds) {
    // exclude react fiber references
    if (edge.name_or_index.toString().startsWith('__reactFiber$')) {
      return false;
    }
    return true;
  }
};
```

Use the leak filter definition in command line interface:
```
memlab find-leaks --leak-filter <PATH TO leak-filter.js>
```

```
memlab run --scenario <SCENARIO FILE> --leak-filter <PATH TO leak-filter.js>
```

Differential Revision: D53167155

fbshipit-source-id: b9006fd309276452acbe216c577f34c28b071ffa

packages/api/src/__tests__/API/E2EFindMemoryLeaks.test.ts

@@ -17,7 +17,12 @@ import os from 'os';
 import path from 'path';
 import fs from 'fs-extra';
 import {run} from '../../index';
-import {scenario, testSetup, testTimeout} from './lib/E2ETestSettings';
+import {
+  getUniqueID,
+  scenario,
+  testSetup,
+  testTimeout,
+} from './lib/E2ETestSettings';
 
 beforeEach(testSetup);
 
@@ -61,39 +66,6 @@ test(
   testTimeout,
 );
 
-test(
-  'self-defined leak detector can find TestObject',
-  async () => {
-    const selfDefinedScenario: IScenario = {
-      app: (): string => 'test-spa',
-      url: (): string => '',
-      action: async (page: Page): Promise<void> =>
-        await page.click('[data-testid="link-4"]'),
-      leakFilter: (node: IHeapNode) => {
-        return node.name === 'TestObject' && node.type === 'object';
-      },
-    };
-
-    const workDir = path.join(os.tmpdir(), 'memlab-api-test', `${process.pid}`);
-    fs.mkdirsSync(workDir);
-
-    const result = await run({
-      scenario: selfDefinedScenario,
-      evalInBrowserAfterInitLoad: injectDetachedDOMElements,
-      workDir,
-    });
-    // detected all different leak trace cluster
-    expect(result.leaks.length).toBe(1);
-    // expect all traces are found
-    expect(
-      result.leaks.some(leak => JSON.stringify(leak).includes('_randomObject')),
-    );
-    const reader = result.runResult;
-    expect(path.resolve(reader.getRootDirectory())).toBe(path.resolve(workDir));
-  },
-  testTimeout,
-);
-
 function injectDetachedDOMElementsWithPrompt() {
   // @ts-ignore
   window.injectHookForLink4 = () => {
@@ -139,7 +111,7 @@ test(
       },
     };
 
-    const workDir = path.join(os.tmpdir(), 'memlab-api-test', `${process.pid}`);
+    const workDir = path.join(os.tmpdir(), 'memlab-api-test', getUniqueID());
     fs.mkdirsSync(workDir);
 
     const result = await run({

packages/api/src/__tests__/API/E2EMemoryLeakFilter.test.ts

@@ -0,0 +1,146 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall web_perf_infra
+ */
+
+/* eslint-disable @typescript-eslint/ban-ts-comment */
+
+import type {Page} from 'puppeteer';
+import type {IHeapEdge, IHeapNode, IScenario} from '@memlab/core';
+
+import os from 'os';
+import path from 'path';
+import fs from 'fs-extra';
+import {run} from '../../index';
+import {getUniqueID, testSetup, testTimeout} from './lib/E2ETestSettings';
+
+beforeEach(testSetup);
+
+function injectDetachedDOMElements() {
+  // @ts-ignore
+  window.injectHookForLink4 = () => {
+    class TestObject {
+      key: 'value';
+    }
+    const arr = [];
+    for (let i = 0; i < 23; ++i) {
+      arr.push(document.createElement('div'));
+    }
+    // @ts-ignore
+    window.__injectedValue = arr;
+    // @ts-ignore
+    window._path_1 = {x: {y: document.createElement('div')}};
+    // @ts-ignore
+    window._path_2 = new Set([document.createElement('div')]);
+    // @ts-ignore
+    window._randomObject = [new TestObject()];
+  };
+}
+
+test(
+  'self-defined leak detector can find TestObject',
+  async () => {
+    const selfDefinedScenario: IScenario = {
+      app: (): string => 'test-spa',
+      url: (): string => '',
+      action: async (page: Page): Promise<void> =>
+        await page.click('[data-testid="link-4"]'),
+      leakFilter: (node: IHeapNode) => {
+        return node.name === 'TestObject' && node.type === 'object';
+      },
+    };
+
+    const workDir = path.join(os.tmpdir(), 'memlab-api-test', getUniqueID());
+    fs.mkdirsSync(workDir);
+
+    const result = await run({
+      scenario: selfDefinedScenario,
+      evalInBrowserAfterInitLoad: injectDetachedDOMElements,
+      workDir,
+    });
+    // detected all different leak trace cluster
+    expect(result.leaks.length).toBe(1);
+    // expect all traces are found
+    expect(
+      result.leaks.some(leak => JSON.stringify(leak).includes('_randomObject')),
+    );
+    const reader = result.runResult;
+    expect(path.resolve(reader.getRootDirectory())).toBe(path.resolve(workDir));
+  },
+  testTimeout,
+);
+
+test(
+  'self-defined retainer trace filter work as expected (part 1)',
+  async () => {
+    const selfDefinedScenario: IScenario = {
+      app: (): string => 'test-spa',
+      url: (): string => '',
+      action: async (page: Page): Promise<void> =>
+        await page.click('[data-testid="link-4"]'),
+      retainerReferenceFilter: (edge: IHeapEdge) => {
+        return edge.name_or_index !== '_path_1';
+      },
+    };
+
+    const workDir = path.join(os.tmpdir(), 'memlab-api-test', getUniqueID());
+    fs.mkdirsSync(workDir);
+
+    const result = await run({
+      scenario: selfDefinedScenario,
+      evalInBrowserAfterInitLoad: injectDetachedDOMElements,
+      workDir,
+    });
+    // detected all different leak trace cluster
+    expect(result.leaks.length).toBe(1);
+    // expect the none of the traces to include _path_1
+    expect(
+      result.leaks.every(leak => !JSON.stringify(leak).includes('_path_1')),
+    );
+    // expect some of the traces to include _path_2
+    expect(result.leaks.some(leak => JSON.stringify(leak).includes('_path_2')));
+    const reader = result.runResult;
+    expect(path.resolve(reader.getRootDirectory())).toBe(path.resolve(workDir));
+  },
+  testTimeout,
+);
+
+test(
+  'self-defined retainer trace filter work as expected (part 2)',
+  async () => {
+    const selfDefinedScenario: IScenario = {
+      app: (): string => 'test-spa',
+      url: (): string => '',
+      action: async (page: Page): Promise<void> =>
+        await page.click('[data-testid="link-4"]'),
+      retainerReferenceFilter: (edge: IHeapEdge) => {
+        return edge.name_or_index !== '_path_2';
+      },
+    };
+
+    const workDir = path.join(os.tmpdir(), 'memlab-api-test', getUniqueID());
+    fs.mkdirsSync(workDir);
+
+    const result = await run({
+      scenario: selfDefinedScenario,
+      evalInBrowserAfterInitLoad: injectDetachedDOMElements,
+      workDir,
+    });
+    // detected all different leak trace cluster
+    expect(result.leaks.length).toBe(1);
+    // expect the none of the traces to include _path_2
+    expect(
+      result.leaks.every(leak => !JSON.stringify(leak).includes('_path_2')),
+    );
+    // expect some of the traces to include _path_1
+    expect(result.leaks.some(leak => JSON.stringify(leak).includes('_path_1')));
+    const reader = result.runResult;
+    expect(path.resolve(reader.getRootDirectory())).toBe(path.resolve(workDir));
+  },
+  testTimeout,
+);

packages/core/src/lib/Config.ts

@@ -619,17 +619,30 @@ export class MemLabConfig {
     if (scenario == null) {
       return;
     }
+    let hasCallback = false;
+    const externalFilter: ILeakFilter = {};
 
     // set leak filter
-    const {leakFilter, beforeLeakFilter} = scenario;
-    if (typeof leakFilter !== 'function') {
-      return;
+    const {leakFilter, beforeLeakFilter, retainerReferenceFilter} = scenario;
+    if (typeof leakFilter === 'function') {
+      hasCallback = true;
+      externalFilter.leakFilter = leakFilter;
     }
-    this.externalLeakFilter = {leakFilter};
 
     // set leak filter init callback
     if (typeof beforeLeakFilter === 'function') {
-      this.externalLeakFilter.beforeLeakFilter = beforeLeakFilter;
+      hasCallback = true;
+      externalFilter.beforeLeakFilter = beforeLeakFilter;
+    }
+
+    // set retainer reference filter callback
+    if (typeof retainerReferenceFilter === 'function') {
+      hasCallback = true;
+      externalFilter.retainerReferenceFilter = retainerReferenceFilter;
+    }
+
+    if (hasCallback) {
+      this.externalLeakFilter = externalFilter;
     }
   }