feat: migration from mswjs to internal solution (#16)

* feat: implement basic http interceptor * feat: adjust supergood to the new interceptor * feat: fixed tests and minor refactoring * test: refactor tests * test: add config for unit tests * refactor: add node 14 support * chore: remove index file for interceptor * 1.1.49-beta.0 * Ignore internal URLs by default * 1.1.49-beta.1 * Hoisting ignored domains * Added metadata to log better * Added space for no reason * 1.1.49-beta.2 * test: add unit tests for interception logic * test: remove redundant test * Add more metadata for debugging (#18) * Adding additional metadata * Add request urls * Fixed up metadata * Add request URL * Add more metadata for debugging * Add partial match for ignored domains (#20) * Adding partial matches for included domains * Added partial string matching for ignored domains * 1.1.49-beta.4 * feat: allow to intercept fetch requests * chore: update scripts * chore: disable logger for a while * refactor: remove redundant logging * test: refactor tests and use new matchers --------- Co-authored-by: Alex Klarfeld <[email protected]> Co-authored-by: Alex Klarfeld <[email protected]>
supergoodsystems · Dec 12, 2023 · 1abeb9f · 1abeb9f
1 parent f0ceb7f
commit 1abeb9f
Show file tree

Hide file tree

Showing 52 changed files with 3,777 additions and 1,679 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "supergood",
-  "version": "1.1.49",
+  "version": "1.1.49-beta.4",
   "description": "",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -29,18 +29,24 @@
     "check:publish-ready": "yarn build && yarn test",
     "preversion": "yarn check:publish-ready",
     "postversion": "git add package.json && git commit -m \"chore: update $npm_package_name to v$npm_package_version\" && git tag $npm_package_name@$npm_package_version",
-    "prepublish": "yarn check:publish-ready",
+    "prepublishOnly": "yarn check:publish-ready",
     "postpublish": "git push origin && git push origin --tags",
-    "test": "NODE_ENV=test jest --setupFiles dotenv/config",
+    "test": "yarn run test:unit && yarn run test:e2e",
+    "test:unit": "jest -c test/jest.unit.config.js",
+    "test:e2e": "node ./test/mock-server & jest -c test/jest.e2e.config.js",
+    "posttest:e2e": "kill -9 $(lsof -t -i:3001)",
     "clean": "rm -rf dist/ && rm -rf supergood-*.log",
-    "build": "yarn run clean && tsc"
+    "build": "yarn run clean && tsc -p ./tsconfig.lib.json"
   },
   "dependencies": {
+    "headers-polyfill": "^4.0.2",
     "lodash.get": "^4.4.2",
     "lodash.set": "^4.3.2",
     "node-cache": "^5.1.2",
+    "pino": "^8.16.2",
     "signal-exit": "^3.0.7",
-    "supergood-interceptors": "0.17.7-rc.0"
+    "ts-essentials": "^9.4.1",
+    "web-encoding": "^1.1.5"
   },
   "devDependencies": {
     "@types/jest": "^29.5.8",
@@ -58,6 +64,7 @@
     "eslint-plugin-jest": "^27.1.3",
     "eslint-plugin-prettier": "^4.2.1",
     "jest": "^29.2.1",
+    "jest-extended": "^4.0.2",
     "json-server": "^0.17.0",
     "openai": "^4.10.0",
     "postgres": "^3.3.4",

diff --git a/src/constants.ts b/src/constants.ts
@@ -2,6 +2,7 @@ const defaultConfig = {
   flushInterval: 1000,
   eventSinkEndpoint: '/events',
   errorSinkEndpoint: '/errors',
+  allowLocalUrls: false,
   keysToHash: [],
   ignoredDomains: [],
 

diff --git a/src/index.ts b/src/index.ts
@@ -1,22 +1,21 @@
-import { IsomorphicRequest } from 'supergood-interceptors';
 import NodeCache from 'node-cache';
+import { serialize } from 'v8';
 import {
   getHeaderOptions,
   logger,
   safeParseJson,
   prepareData,
-  shouldCachePayload,
   sleep
 } from './utils';
 import { postEvents } from './api';
 
-import { ClientRequestInterceptor } from 'supergood-interceptors/lib/interceptors/ClientRequest';
 import {
   HeaderOptionType,
   EventRequestType,
   ConfigType,
   LoggerType,
-  RequestType
+  RequestType,
+  MetadataType
 } from './types';
 import {
   defaultConfig,
@@ -26,13 +25,19 @@ import {
   LocalClientSecret
 } from './constants';
 import onExit from 'signal-exit';
+import { NodeRequestInterceptor } from './interceptor/NodeRequestInterceptor';
+import { IsomorphicRequest } from './interceptor/utils/IsomorphicRequest';
+import { IsomorphicResponse } from './interceptor/utils/IsomorphicResponse';
+import { BatchInterceptor } from './interceptor/BatchInterceptor';
+import { FetchInterceptor } from './interceptor/FetchInterceptor';
 
 const Supergood = () => {
   let eventSinkUrl: string;
   let errorSinkUrl: string;
 
   let headerOptions: HeaderOptionType;
   let supergoodConfig: ConfigType;
+  let supergoodMetadata: MetadataType;
 
   let requestCache: NodeCache;
   let responseCache: NodeCache;
@@ -42,21 +47,24 @@ const Supergood = () => {
 
   let localOnly = false;
 
-  let interceptor: ClientRequestInterceptor;
+  let interceptor: BatchInterceptor;
 
   const init = async (
     {
       clientId,
       clientSecret,
-      config
+      config,
+      metadata
     }: {
       clientId?: string;
       clientSecret?: string;
       config?: Partial<ConfigType>;
+      metadata?: Partial<MetadataType>;
     } = {
       clientId: process.env.SUPERGOOD_CLIENT_ID as string,
       clientSecret: process.env.SUPERGOOD_CLIENT_SECRET as string,
-      config: {} as Partial<ConfigType>
+      config: {} as Partial<ConfigType>,
+      metadata: {} as Partial<MetadataType>
     },
     baseUrl = process.env.SUPERGOOD_BASE_URL || 'https://api.supergood.ai'
   ) => {
@@ -71,110 +79,140 @@ const Supergood = () => {
       ...defaultConfig,
       ...config
     } as ConfigType;
+    supergoodMetadata = metadata as MetadataType;
 
     requestCache = new NodeCache({
       stdTTL: 0
     });
     responseCache = new NodeCache({
       stdTTL: 0
     });
-    interceptor = new ClientRequestInterceptor({
-      ignoredDomains: supergoodConfig.ignoredDomains
-    });
+    const interceptorOpts = {
+      ignoredDomains: supergoodConfig.ignoredDomains,
+      allowLocalUrls: supergoodConfig.allowLocalUrls,
+      baseUrl
+    };
+
+    interceptor = new BatchInterceptor([
+      new NodeRequestInterceptor(interceptorOpts),
+      ...(FetchInterceptor.checkEnvironment()
+        ? [new FetchInterceptor(interceptorOpts)]
+        : [])
+    ]);
 
     errorSinkUrl = `${baseUrl}${supergoodConfig.errorSinkEndpoint}`;
     eventSinkUrl = `${baseUrl}${supergoodConfig.eventSinkEndpoint}`;
 
     headerOptions = getHeaderOptions(clientId, clientSecret);
     log = logger({ errorSinkUrl, headerOptions });
 
-    interceptor.apply();
-    interceptor.on('request', async (request: IsomorphicRequest) => {
-      const requestId = request.id;
-      try {
-        const url = new URL(request.url);
-        // Meant for debug and testing purposes
-        if (url.pathname === TestErrorPath) {
-          throw new Error(errors.TEST_ERROR);
-        }
+    interceptor.setup();
 
-        const body = await request.clone().text();
-        const requestData = {
-          id: requestId,
-          headers: Object.fromEntries(request.headers.entries()),
-          method: request.method,
-          url: url.href,
-          path: url.pathname,
-          search: url.search,
-          body: safeParseJson(body),
-          requestedAt: new Date()
-        } as RequestType;
-
-        cacheRequest(requestData, baseUrl);
-      } catch (e) {
-        log.error(
-          errors.CACHING_REQUEST,
-          { config: supergoodConfig },
-          e as Error,
-          {
-            reportOut: !localOnly
+    interceptor.on(
+      'request',
+      async (request: IsomorphicRequest, requestId: string) => {
+        try {
+          const url = new URL(request.url);
+          // Meant for debug and testing purposes
+
+          if (url.pathname === TestErrorPath) {
+            throw new Error(errors.TEST_ERROR);
           }
-        );
-      }
-    });
 
-    interceptor.on('response', async (request, response) => {
-      const requestId = request.id;
-      try {
-        const requestData = requestCache.get(requestId) as {
-          request: RequestType;
-        };
-        if (requestData) {
-          const responseData = {
-            response: {
-              headers: Object.fromEntries(response.headers.entries()),
-              status: response.status,
-              statusText: response.statusText,
-              body: response.body && safeParseJson(response.body),
-              respondedAt: new Date()
+          const body = await request.clone().text();
+          const requestData = {
+            id: requestId,
+            headers: Object.fromEntries(request.headers.entries()),
+            method: request.method,
+            url: url.href,
+            path: url.pathname,
+            search: url.search,
+            body: safeParseJson(body),
+            requestedAt: new Date()
+          } as RequestType;
+
+          cacheRequest(requestData, baseUrl);
+        } catch (e) {
+          log.error(
+            errors.CACHING_REQUEST,
+            {
+              config: supergoodConfig,
+              metadata: {
+                requestUrl: request.url.toString(),
+                payloadSize: serialize(request).length,
+                ...supergoodMetadata
+              }
             },
-            ...requestData
-          } as EventRequestType;
-          cacheResponse(responseData, baseUrl);
+            e as Error,
+            {
+              reportOut: !localOnly
+            }
+          );
         }
-      } catch (e) {
-        log.error(
-          errors.CACHING_RESPONSE,
-          { config: supergoodConfig },
-          e as Error
-        );
       }
-    });
+    );
+
+    interceptor.on(
+      'response',
+      async (response: IsomorphicResponse, requestId: string) => {
+        let requestData = { url: '' };
+        let responseData = {};
+
+        try {
+          const requestData = requestCache.get(requestId) as {
+            request: RequestType;
+          };
+
+          if (requestData) {
+            const responseData = {
+              response: {
+                headers: Object.fromEntries(response.headers.entries()),
+                status: response.status,
+                statusText: response.statusText,
+                body: response.body && safeParseJson(response.body),
+                respondedAt: new Date()
+              },
+              ...requestData
+            } as EventRequestType;
+            cacheResponse(responseData, baseUrl);
+          }
+        } catch (e) {
+          log.error(
+            errors.CACHING_RESPONSE,
+            {
+              config: supergoodConfig,
+              metadata: {
+                ...supergoodMetadata,
+                requestUrl: requestData.url,
+                payloadSize: responseData ? serialize(responseData).length : 0
+              }
+            },
+            e as Error
+          );
+        }
+      }
+    );
 
     // Flushes the cache every <flushInterval> milliseconds
     interval = setInterval(flushCache, supergoodConfig.flushInterval);
     interval.unref();
   };
 
   const cacheRequest = async (request: RequestType, baseUrl: string) => {
-    if (shouldCachePayload(request.url, baseUrl)) {
-      requestCache.set(request.id, { request });
-      log.debug('Setting Request Cache', {
-        request
-      });
-    }
+    requestCache.set(request.id, { request });
+    log.debug('Setting Request Cache', {
+      request
+    });
   };
 
   const cacheResponse = async (event: EventRequestType, baseUrl: string) => {
-    if (shouldCachePayload(event.request.url, baseUrl)) {
-      responseCache.set(event.request.id, event);
-      log.debug('Setting Response Cache', {
-        id: event.request.id,
-        ...event
-      });
-      requestCache.del(event.request.id);
-      log.debug('Deleting Request Cache', { id: event.request.id });
-    }
+    responseCache.set(event.request.id, event);
+    log.debug('Setting Response Cache', {
+      id: event.request.id,
+      ...event
+    });
+    requestCache.del(event.request.id);
+    log.debug('Deleting Request Cache', { id: event.request.id });
   };
 
   // Force flush cache means don't wait for responses
@@ -216,18 +254,31 @@ const Supergood = () => {
       if (error.message === errors.UNAUTHORIZED) {
         log.error(
           errors.UNAUTHORIZED,
-          { config: supergoodConfig },
+          {
+            config: supergoodConfig,
+            metadata: {
+              ...supergoodMetadata
+            }
+          },
           error,
           {
             reportOut: false
           }
         );
         clearInterval(interval);
-        interceptor.dispose();
+        interceptor.teardown();
       } else {
         log.error(
           errors.POSTING_EVENTS,
-          { config: supergoodConfig },
+          {
+            config: supergoodConfig,
+            metadata: {
+              numberOfEvents: data.length,
+              payloadSize: serialize(data).length,
+              requestUrls: data.map((event) => event?.request?.url),
+              ...supergoodMetadata
+            }
+          },
           error,
           {
             reportOut: !localOnly
@@ -253,7 +304,7 @@ const Supergood = () => {
       await sleep(supergoodConfig.waitAfterClose);
     }
 
-    interceptor.dispose();
+    interceptor.teardown();
     await flushCache({ force });
     return false;
   };