From b5b4da4349efc9dc702555647d1adf91ed1fe4ee Mon Sep 17 00:00:00 2001 From: Alex Klarfeld <1866143+aklarfeld@users.noreply.github.com> Date: Tue, 2 Jan 2024 11:21:35 -0800 Subject: [PATCH] Post telemetry stats (#23) * Add telemetry posting * Post telemetry stats * Added cacheKeys and cacheSize --- src/api.ts | 17 +++++++++++-- src/constants.ts | 2 ++ src/index.ts | 41 ++++++++++++++++++++++++++------ src/types.ts | 14 ++++++++--- src/utils.ts | 5 ++-- test/e2e/telemetry.e2e.test.ts | 36 ++++++++++++++++++++++++++++ test/utils/function-call-args.ts | 10 +++++++- test/utils/mock-api.ts | 6 ++++- 8 files changed, 115 insertions(+), 16 deletions(-) create mode 100644 test/e2e/telemetry.e2e.test.ts diff --git a/src/api.ts b/src/api.ts index 5a4eb0f..1e7672f 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,4 +1,4 @@ -import { HeaderOptionType, EventRequestType, ErrorPayloadType } from './types'; +import { HeaderOptionType, EventRequestType, ErrorPayloadType, TelemetryType } from './types'; import { post, get } from './utils'; const postError = async ( @@ -32,9 +32,22 @@ const postEvents = async ( return response; }; +const postTelemetry = async ( + telemetryUrl: string, + data: TelemetryType, + options: HeaderOptionType +) => { + const response = await post( + telemetryUrl, + data, + options.headers.Authorization + ); + return response; +} + const fetchRemoteConfig = async (configUrl: string, options: HeaderOptionType) => { const response = await get(configUrl, options.headers.Authorization); return JSON.parse(response); } -export { postError, postEvents, fetchRemoteConfig }; +export { postError, postEvents, fetchRemoteConfig, postTelemetry }; diff --git a/src/constants.ts b/src/constants.ts index a711360..5dc4332 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -4,6 +4,7 @@ const defaultConfig = { eventSinkEndpoint: '/events', errorSinkEndpoint: '/errors', remoteConfigFetchEndpoint: '/config', + telemetryEndpoint: '/telemetry', allowLocalUrls: false, ignoredDomains: [], @@ -18,6 +19,7 @@ const errors = { DUMPING_DATA_TO_DISK: 'Error Dumping Data to Disk', POSTING_EVENTS: 'Error Posting Events', POSTING_ERRORS: 'Error Posting Errors', + POSTING_TELEMETRY: 'Error Posting Telemetry', FETCHING_CONFIG: 'Error Fetching Config', WRITING_TO_DISK: 'Error writing to disk', TEST_ERROR: 'Test Error for Testing Purposes', diff --git a/src/index.ts b/src/index.ts index 705e74d..e4e1e8b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,7 @@ import { processRemoteConfig, getEndpointConfigForRequest } from './utils'; -import { postEvents, fetchRemoteConfig } from './api'; +import { postEvents, fetchRemoteConfig, postTelemetry } from './api'; import v8 from 'v8'; import { HeaderOptionType, @@ -37,6 +37,7 @@ const Supergood = () => { let eventSinkUrl: string; let errorSinkUrl: string; let remoteConfigFetchUrl: string; + let telemetryUrl: string; let headerOptions: HeaderOptionType; let supergoodConfig: ConfigType; @@ -107,6 +108,7 @@ const Supergood = () => { errorSinkUrl = `${baseUrl}${supergoodConfig.errorSinkEndpoint}`; eventSinkUrl = `${baseUrl}${supergoodConfig.eventSinkEndpoint}`; remoteConfigFetchUrl = `${baseUrl}${supergoodConfig.remoteConfigFetchEndpoint}`; + telemetryUrl = `${baseUrl}${supergoodConfig.telemetryEndpoint}`; headerOptions = getHeaderOptions(clientId, clientSecret); log = logger({ errorSinkUrl, headerOptions }); @@ -162,8 +164,8 @@ const Supergood = () => { config: supergoodConfig, metadata: { requestUrl: request.url.toString(), - payloadSize: serialize(request).length, - ...supergoodMetadata + size: serialize(request).length, + ...supergoodMetadata, } }, e as Error, @@ -211,9 +213,9 @@ const Supergood = () => { { config: supergoodConfig, metadata: { - ...supergoodMetadata, requestUrl: requestData.url, - payloadSize: responseData ? serialize(responseData).length : 0 + size: responseData ? serialize(responseData).length : 0, + ...supergoodMetadata } }, e as Error @@ -281,6 +283,31 @@ const Supergood = () => { return; } + const { keys, vsize } = responseCache.getStats(); + + try { + // Post the telemetry after the events make it, but before we delete the cache + await postTelemetry(telemetryUrl, { cacheKeys: keys, cacheSize: vsize, ...supergoodMetadata }, headerOptions); + } catch (e) { + const error = e as Error; + log.error( + errors.POSTING_TELEMETRY, + { + config: supergoodConfig, + metadata: { + keys, + size: vsize, + requestUrls: data.map((event) => event?.request?.url), + ...supergoodMetadata + } + }, + error, + { + reportOut: !localOnly + } + ) + } + try { if (localOnly) { log.debug(JSON.stringify(data, null, 2), { force }); @@ -310,8 +337,8 @@ const Supergood = () => { { config: supergoodConfig, metadata: { - numberOfEvents: data.length, - payloadSize: serialize(data).length, + keys: data.length, + size: serialize(data).length, requestUrls: data.map((event) => event?.request?.url), ...supergoodMetadata } diff --git a/src/types.ts b/src/types.ts index badd8aa..7ae7dce 100644 --- a/src/types.ts +++ b/src/types.ts @@ -44,10 +44,17 @@ interface ConfigType { remoteConfigFetchEndpoint: string; // Defaults to {baseUrl}/config if not provided eventSinkEndpoint: string; // Defaults to {baseUrl}/events if not provided errorSinkEndpoint: string; // Defaults to {baseUrl}/errors if not provided + telemetryEndpoint: string; // Defaults to {baseUrl}/telemetry if not provided waitAfterClose: number; remoteConfig: RemoteConfigType; } +interface TelemetryType { + cacheKeys: number; + cacheSize: number; + serviceName?: string; +} + interface EndpointConfigType { location: string; regex: string; @@ -62,8 +69,8 @@ interface RemoteConfigType { }; interface MetadataType { - numberOfEvents?: number; - payloadSize?: number; + keys?: number; + size?: number; requestUrls?: string[]; requestUrl?: string; serviceName?: string; @@ -149,5 +156,6 @@ export type { RemoteConfigType, EndpointConfigType, RemoteConfigPayloadType, - MetadataType + MetadataType, + TelemetryType }; diff --git a/src/utils.ts b/src/utils.ts index 3deefc2..e9f3eb3 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -8,7 +8,8 @@ import { RemoteConfigPayloadType, RemoteConfigType, EndpointConfigType, - SensitiveKeyMetadata + SensitiveKeyMetadata, + TelemetryType } from './types'; import { postError } from './api'; import { name, version } from '../package.json'; @@ -229,7 +230,7 @@ const getByteSize = (s: string) => { const post = ( url: string, - data: Array | ErrorPayloadType, + data: Array | ErrorPayloadType | TelemetryType, authorization: string ): Promise => { const dataString = JSON.stringify(data); diff --git a/test/e2e/telemetry.e2e.test.ts b/test/e2e/telemetry.e2e.test.ts new file mode 100644 index 0000000..68af722 --- /dev/null +++ b/test/e2e/telemetry.e2e.test.ts @@ -0,0 +1,36 @@ +import Supergood from '../../src'; +import { mockApi } from '../utils/mock-api'; +import axios from 'axios'; + +import { + MOCK_DATA_SERVER, + SUPERGOOD_CLIENT_ID, + SUPERGOOD_CLIENT_SECRET, + SUPERGOOD_CONFIG, + SUPERGOOD_SERVER +} from '../consts'; +import { getTelemetry } from '../utils/function-call-args'; + +describe('telemetry posting', () => { + const { postTelemetryMock } = mockApi(); + it('should accurately post telemetry', async () => { + await Supergood.init( + { + config: { ...SUPERGOOD_CONFIG, allowLocalUrls: true }, + clientId: SUPERGOOD_CLIENT_ID, + clientSecret: SUPERGOOD_CLIENT_SECRET, + metadata: { + serviceName: "test-service-name", + } + }, + SUPERGOOD_SERVER + ); + await axios.get(`${MOCK_DATA_SERVER}/posts`); + await Supergood.close(); + const { cacheKeys, cacheSize, serviceName } = getTelemetry(postTelemetryMock); + expect(cacheKeys).toEqual(1); + expect(cacheSize).toEqual(160); + expect(serviceName).toEqual("test-service-name"); + }) + +}) diff --git a/test/utils/function-call-args.ts b/test/utils/function-call-args.ts index 7d57f30..062723c 100644 --- a/test/utils/function-call-args.ts +++ b/test/utils/function-call-args.ts @@ -1,4 +1,4 @@ -import { ErrorPayloadType, EventRequestType } from '../../src/types'; +import { ErrorPayloadType, EventRequestType, TelemetryType } from '../../src/types'; export const getEvents = ( mockedPostEvents: jest.SpyInstance @@ -16,6 +16,14 @@ export const getErrors = ( )[1] as ErrorPayloadType; }; +export const getTelemetry = ( + mockedPostTelemetry: jest.SpyInstance +): TelemetryType => { + return Object.values( + mockedPostTelemetry.mock.calls.flat() + )[1] as TelemetryType; +}; + export function checkPostedEvents( instance: jest.SpyInstance, eventsCount: number, diff --git a/test/utils/mock-api.ts b/test/utils/mock-api.ts index 00d4300..e38a7ca 100644 --- a/test/utils/mock-api.ts +++ b/test/utils/mock-api.ts @@ -27,5 +27,9 @@ export const mockApi = ( .spyOn(api, 'fetchRemoteConfig') .mockImplementation(fetchRemoteConfigFunction ?? (async () => fetchRemoteConfigResponse ?? ([] as any))); - return { postEventsMock, postErrorMock, fetchRemoteConfigMock }; + const postTelemetryMock = jest + .spyOn(api, 'postTelemetry') + .mockImplementation((async (_, payload) => ({ payload } as any))); + + return { postEventsMock, postErrorMock, fetchRemoteConfigMock, postTelemetryMock }; }