Post telemetry stats (#23)

* Add telemetry posting

* Post telemetry stats

* Added cacheKeys and cacheSize

src/api.ts

@@ -1,4 +1,4 @@
-import { HeaderOptionType, EventRequestType, ErrorPayloadType } from './types';
+import { HeaderOptionType, EventRequestType, ErrorPayloadType, TelemetryType } from './types';
 import { post, get } from './utils';
 
 const postError = async (
@@ -32,9 +32,22 @@ const postEvents = async (
   return response;
 };
 
+const postTelemetry = async (
+  telemetryUrl: string,
+  data: TelemetryType,
+  options: HeaderOptionType
+) => {
+  const response = await post(
+    telemetryUrl,
+    data,
+    options.headers.Authorization
+  );
+  return response;
+}
+
 const fetchRemoteConfig = async (configUrl: string, options: HeaderOptionType) => {
   const response = await get(configUrl, options.headers.Authorization);
   return JSON.parse(response);
 }
 
-export { postError, postEvents, fetchRemoteConfig };
+export { postError, postEvents, fetchRemoteConfig, postTelemetry };

src/constants.ts

@@ -4,6 +4,7 @@ const defaultConfig = {
   eventSinkEndpoint: '/events',
   errorSinkEndpoint: '/errors',
   remoteConfigFetchEndpoint: '/config',
+  telemetryEndpoint: '/telemetry',
   allowLocalUrls: false,
   ignoredDomains: [],
 
@@ -18,6 +19,7 @@ const errors = {
   DUMPING_DATA_TO_DISK: 'Error Dumping Data to Disk',
   POSTING_EVENTS: 'Error Posting Events',
   POSTING_ERRORS: 'Error Posting Errors',
+  POSTING_TELEMETRY: 'Error Posting Telemetry',
   FETCHING_CONFIG: 'Error Fetching Config',
   WRITING_TO_DISK: 'Error writing to disk',
   TEST_ERROR: 'Test Error for Testing Purposes',

src/index.ts

@@ -9,7 +9,7 @@ import {
   processRemoteConfig,
   getEndpointConfigForRequest
 } from './utils';
-import { postEvents, fetchRemoteConfig } from './api';
+import { postEvents, fetchRemoteConfig, postTelemetry } from './api';
 import v8 from 'v8';
 import {
   HeaderOptionType,
@@ -37,6 +37,7 @@ const Supergood = () => {
   let eventSinkUrl: string;
   let errorSinkUrl: string;
   let remoteConfigFetchUrl: string;
+  let telemetryUrl: string;
 
   let headerOptions: HeaderOptionType;
   let supergoodConfig: ConfigType;
@@ -107,6 +108,7 @@ const Supergood = () => {
     errorSinkUrl = `${baseUrl}${supergoodConfig.errorSinkEndpoint}`;
     eventSinkUrl = `${baseUrl}${supergoodConfig.eventSinkEndpoint}`;
     remoteConfigFetchUrl = `${baseUrl}${supergoodConfig.remoteConfigFetchEndpoint}`;
+    telemetryUrl = `${baseUrl}${supergoodConfig.telemetryEndpoint}`;
 
     headerOptions = getHeaderOptions(clientId, clientSecret);
     log = logger({ errorSinkUrl, headerOptions });
@@ -162,8 +164,8 @@ const Supergood = () => {
                 config: supergoodConfig,
                 metadata: {
                   requestUrl: request.url.toString(),
-                  payloadSize: serialize(request).length,
-                  ...supergoodMetadata
+                  size: serialize(request).length,
+                  ...supergoodMetadata,
                 }
               },
               e as Error,
@@ -211,9 +213,9 @@ const Supergood = () => {
               {
                 config: supergoodConfig,
                 metadata: {
-                  ...supergoodMetadata,
                   requestUrl: requestData.url,
-                  payloadSize: responseData ? serialize(responseData).length : 0
+                  size: responseData ? serialize(responseData).length : 0,
+                  ...supergoodMetadata
                 }
               },
               e as Error
@@ -281,6 +283,31 @@ const Supergood = () => {
       return;
     }
 
+    const { keys, vsize } = responseCache.getStats();
+
+    try {
+      // Post the telemetry after the events make it, but before we delete the cache
+      await postTelemetry(telemetryUrl, { cacheKeys: keys, cacheSize: vsize, ...supergoodMetadata }, headerOptions);
+    } catch (e) {
+      const error = e as Error;
+      log.error(
+        errors.POSTING_TELEMETRY,
+        {
+          config: supergoodConfig,
+          metadata: {
+            keys,
+            size: vsize,
+            requestUrls: data.map((event) => event?.request?.url),
+            ...supergoodMetadata
+          }
+        },
+        error,
+        {
+          reportOut: !localOnly
+        }
+      )
+    }
+
     try {
       if (localOnly) {
         log.debug(JSON.stringify(data, null, 2), { force });
@@ -310,8 +337,8 @@ const Supergood = () => {
           {
             config: supergoodConfig,
             metadata: {
-              numberOfEvents: data.length,
-              payloadSize: serialize(data).length,
+              keys: data.length,
+              size: serialize(data).length,
               requestUrls: data.map((event) => event?.request?.url),
               ...supergoodMetadata
             }

src/types.ts

@@ -44,10 +44,17 @@ interface ConfigType {
   remoteConfigFetchEndpoint: string; // Defaults to {baseUrl}/config if not provided
   eventSinkEndpoint: string; // Defaults to {baseUrl}/events if not provided
   errorSinkEndpoint: string; // Defaults to {baseUrl}/errors if not provided
+  telemetryEndpoint: string; // Defaults to {baseUrl}/telemetry if not provided
   waitAfterClose: number;
   remoteConfig: RemoteConfigType;
 }
 
+interface TelemetryType {
+  cacheKeys: number;
+  cacheSize: number;
+  serviceName?: string;
+}
+
 interface EndpointConfigType {
   location: string;
   regex: string;
@@ -62,8 +69,8 @@ interface RemoteConfigType {
 };
 
 interface MetadataType {
-  numberOfEvents?: number;
-  payloadSize?: number;
+  keys?: number;
+  size?: number;
   requestUrls?: string[];
   requestUrl?: string;
   serviceName?: string;
@@ -149,5 +156,6 @@ export type {
   RemoteConfigType,
   EndpointConfigType,
   RemoteConfigPayloadType,
-  MetadataType
+  MetadataType,
+  TelemetryType
 };

src/utils.ts

@@ -8,7 +8,8 @@ import {
   RemoteConfigPayloadType,
   RemoteConfigType,
   EndpointConfigType,
-  SensitiveKeyMetadata
+  SensitiveKeyMetadata,
+  TelemetryType
 } from './types';
 import { postError } from './api';
 import { name, version } from '../package.json';
@@ -229,7 +230,7 @@ const getByteSize = (s: string) => {
 
 const post = (
   url: string,
-  data: Array<EventRequestType> | ErrorPayloadType,
+  data: Array<EventRequestType> | ErrorPayloadType | TelemetryType,
   authorization: string
 ): Promise<string> => {
   const dataString = JSON.stringify(data);

test/e2e/telemetry.e2e.test.ts

@@ -0,0 +1,36 @@
+import Supergood from '../../src';
+import { mockApi } from '../utils/mock-api';
+import axios from 'axios';
+
+import {
+  MOCK_DATA_SERVER,
+  SUPERGOOD_CLIENT_ID,
+  SUPERGOOD_CLIENT_SECRET,
+  SUPERGOOD_CONFIG,
+  SUPERGOOD_SERVER
+} from '../consts';
+import { getTelemetry } from '../utils/function-call-args';
+
+describe('telemetry posting', () => {
+  const { postTelemetryMock } = mockApi();
+  it('should accurately post telemetry', async () => {
+    await Supergood.init(
+      {
+        config: { ...SUPERGOOD_CONFIG, allowLocalUrls: true },
+        clientId: SUPERGOOD_CLIENT_ID,
+        clientSecret: SUPERGOOD_CLIENT_SECRET,
+        metadata: {
+          serviceName: "test-service-name",
+        }
+      },
+      SUPERGOOD_SERVER
+    );
+    await axios.get(`${MOCK_DATA_SERVER}/posts`);
+    await Supergood.close();
+    const { cacheKeys, cacheSize, serviceName } = getTelemetry(postTelemetryMock);
+    expect(cacheKeys).toEqual(1);
+    expect(cacheSize).toEqual(160);
+    expect(serviceName).toEqual("test-service-name");
+  })
+
+})

test/utils/function-call-args.ts

@@ -1,4 +1,4 @@
-import { ErrorPayloadType, EventRequestType } from '../../src/types';
+import { ErrorPayloadType, EventRequestType, TelemetryType } from '../../src/types';
 
 export const getEvents = (
   mockedPostEvents: jest.SpyInstance
@@ -16,6 +16,14 @@ export const getErrors = (
   )[1] as ErrorPayloadType;
 };
 
+export const getTelemetry = (
+  mockedPostTelemetry: jest.SpyInstance
+): TelemetryType => {
+  return Object.values(
+    mockedPostTelemetry.mock.calls.flat()
+  )[1] as TelemetryType;
+};
+
 export function checkPostedEvents(
   instance: jest.SpyInstance,
   eventsCount: number,

test/utils/mock-api.ts

@@ -27,5 +27,9 @@ export const mockApi = (
     .spyOn(api, 'fetchRemoteConfig')
     .mockImplementation(fetchRemoteConfigFunction ?? (async () => fetchRemoteConfigResponse ?? ([] as any)));
 
-  return { postEventsMock, postErrorMock, fetchRemoteConfigMock };
+  const postTelemetryMock = jest
+    .spyOn(api, 'postTelemetry')
+    .mockImplementation((async (_, payload) => ({ payload } as any)));
+
+  return { postEventsMock, postErrorMock, fetchRemoteConfigMock, postTelemetryMock };
 }