From 793befd395d8fb9ad664f07cd8710a9d51456c50 Mon Sep 17 00:00:00 2001 From: zbenamram <55831041+zbenamram@users.noreply.github.com> Date: Thu, 25 Jul 2024 13:50:56 -0700 Subject: [PATCH] [Feature] - Adding support for method categorization (#52) * adding support for method categorization * tolower --- src/types.ts | 19 ++-- src/utils.test.ts | 169 ++++++++++++++++++++++------- src/utils.ts | 155 ++++++++++++++++---------- test/e2e/remote-config.e2e.test.ts | 3 + 4 files changed, 240 insertions(+), 106 deletions(-) diff --git a/src/types.ts b/src/types.ts index d471919..bec6ffa 100644 --- a/src/types.ts +++ b/src/types.ts @@ -13,7 +13,7 @@ interface JSONObject { [key: string]: JSONValue; } -type BodyType = JSONObject +type BodyType = JSONObject; interface RequestType { id: string; @@ -68,16 +68,17 @@ interface TelemetryType { interface EndpointConfigType { location: string; + method: string; regex: string; ignored: boolean; - sensitiveKeys: Array<{ keyPath: string, action: string }>; + sensitiveKeys: Array<{ keyPath: string; action: string }>; } interface RemoteConfigType { [domain: string]: { [endpointName: string]: EndpointConfigType; }; -}; +} interface MetadataType { keys?: number; @@ -141,18 +142,18 @@ type RemoteConfigPayloadType = Array<{ domain: string; endpoints: Array<{ name: string; + method: string; matchingRegex: { regex: string; location: string; }; endpointConfiguration: { action: string; - sensitiveKeys: Array< - { - keyPath: string; - action: string; - }>; - } + sensitiveKeys: Array<{ + keyPath: string; + action: string; + }>; + }; }>; }>; diff --git a/src/utils.test.ts b/src/utils.test.ts index 88e299a..de0fed4 100644 --- a/src/utils.test.ts +++ b/src/utils.test.ts @@ -4,7 +4,11 @@ import { expandSensitiveKeySetForArrays, redactValuesFromKeys } from './utils'; -import { defaultConfig, SensitiveKeyActions, EndpointActions } from './constants'; +import { + defaultConfig, + SensitiveKeyActions, + EndpointActions +} from './constants'; import { get as _get } from 'lodash'; it('generates multiple sensitive key paths for an array', () => { @@ -35,13 +39,17 @@ it('generates multiple sensitive key paths for an array', () => { ] } }; - const sensitiveKeys = [{ keyPath: 'blog.posts[].title', action: SensitiveKeyActions.REDACT}]; - expect(expandSensitiveKeySetForArrays(obj, sensitiveKeys)).toEqual([ - 'blog.posts[0].title', - 'blog.posts[1].title', - 'blog.posts[2].title', - 'blog.posts[3].title' - ].map((key) => ({ keyPath: key, action: SensitiveKeyActions.REDACT }))); + const sensitiveKeys = [ + { keyPath: 'blog.posts[].title', action: SensitiveKeyActions.REDACT } + ]; + expect(expandSensitiveKeySetForArrays(obj, sensitiveKeys)).toEqual( + [ + 'blog.posts[0].title', + 'blog.posts[1].title', + 'blog.posts[2].title', + 'blog.posts[3].title' + ].map((key) => ({ keyPath: key, action: SensitiveKeyActions.REDACT })) + ); }); it('generates multiple sensitive key paths for an object with nested arrays', () => { @@ -130,19 +138,26 @@ it('generates multiple sensitive key paths for an object with nested arrays', () ] } }; - const sensitiveKeys = [{ keyPath: 'blog.posts[].comments[].body', action: SensitiveKeyActions.REDACT }]; - expect(expandSensitiveKeySetForArrays(obj, sensitiveKeys)).toEqual([ - 'blog.posts[0].comments[0].body', - 'blog.posts[0].comments[1].body', - 'blog.posts[0].comments[2].body', - 'blog.posts[0].comments[3].body', - 'blog.posts[1].comments[0].body', - 'blog.posts[1].comments[1].body', - 'blog.posts[1].comments[2].body', - 'blog.posts[2].comments[0].body', - 'blog.posts[2].comments[1].body', - 'blog.posts[3].comments[0].body' - ].map((key) => ({ keyPath: key, action: SensitiveKeyActions.REDACT }))); + const sensitiveKeys = [ + { + keyPath: 'blog.posts[].comments[].body', + action: SensitiveKeyActions.REDACT + } + ]; + expect(expandSensitiveKeySetForArrays(obj, sensitiveKeys)).toEqual( + [ + 'blog.posts[0].comments[0].body', + 'blog.posts[0].comments[1].body', + 'blog.posts[0].comments[2].body', + 'blog.posts[0].comments[3].body', + 'blog.posts[1].comments[0].body', + 'blog.posts[1].comments[1].body', + 'blog.posts[1].comments[2].body', + 'blog.posts[2].comments[0].body', + 'blog.posts[2].comments[1].body', + 'blog.posts[3].comments[0].body' + ].map((key) => ({ keyPath: key, action: SensitiveKeyActions.REDACT })) + ); }); it('redacts values from keys with proper marshalling', () => { @@ -189,8 +204,14 @@ it('redacts values from keys with proper marshalling', () => { '/posts': { location: 'path', regex: '/posts', + method: 'GET', ignored: false, - sensitiveKeys: [{ keyPath: 'requestBody.posts[].title', action: SensitiveKeyActions.REDACT }] + sensitiveKeys: [ + { + keyPath: 'requestBody.posts[].title', + action: SensitiveKeyActions.REDACT + } + ] } } }; @@ -307,8 +328,14 @@ it('redacts values from keys of nested array', () => { '/posts': { location: 'path', regex: '/posts', + method: 'GET', ignored: false, - sensitiveKeys: [{ keyPath: 'requestBody.posts[].comments[].body', action: SensitiveKeyActions.REDACT }] + sensitiveKeys: [ + { + keyPath: 'requestBody.posts[].comments[].body', + action: SensitiveKeyActions.REDACT + } + ] } } }; @@ -347,8 +374,14 @@ it('will not blow up or redact anything if the sensitive key is bad', () => { '/posts': { location: 'path', regex: '/posts', + method: 'GET', ignored: false, - sensitiveKeys: [{ keyPath: 'request_body.posts[].title[]', action: SensitiveKeyActions.REDACT }] + sensitiveKeys: [ + { + keyPath: 'request_body.posts[].title[]', + action: SensitiveKeyActions.REDACT + } + ] } } }; @@ -396,10 +429,17 @@ it('will prepare the data appropriately for posting to the server', () => { '/posts': { location: 'path', regex: '/posts', + method: 'GET', ignored: false, sensitiveKeys: [ - { keyPath: 'responseBody.user.email', action: SensitiveKeyActions.REDACT}, - { keyPath: 'requestBody.blogType.name', action: SensitiveKeyActions.REDACT} + { + keyPath: 'responseBody.user.email', + action: SensitiveKeyActions.REDACT + }, + { + keyPath: 'requestBody.blogType.name', + action: SensitiveKeyActions.REDACT + } ] } } @@ -439,7 +479,10 @@ it('will force redact all keys if the config is set to do so', () => { name: 'John Doe', email: 'john@doe.com' }, - comments: [{ id: 7, comment: 'good blog'}, { id: 8, comment: 'bad blog'}] + comments: [ + { id: 7, comment: 'good blog' }, + { id: 8, comment: 'bad blog' } + ] } } }; @@ -447,13 +490,18 @@ it('will force redact all keys if the config is set to do so', () => { [new URL(MOCK_DATA_SERVER).hostname]: { '/posts': { location: 'path', + method: 'GET', regex: '/posts', ignored: false, sensitiveKeys: [] } } }; - const config = { remoteConfig, ...defaultConfig, forceRedactAll: true } as ConfigType; + const config = { + remoteConfig, + ...defaultConfig, + forceRedactAll: true + } as ConfigType; const events = prepareData([obj], config); expect(_get(events[0], 'request.body.blogType.name')).toBeFalsy(); expect(_get(events[0], 'response.body.name')).toBeFalsy(); @@ -494,7 +542,10 @@ it('will redact by default if the config is set to do so', () => { name: 'John Doe', email: 'john@doe.com' }, - comments: [{ id: 7, comment: 'good blog'}, { id: 8, comment: 'bad blog'}] + comments: [ + { id: 7, comment: 'good blog' }, + { id: 8, comment: 'bad blog' } + ] } } }; @@ -502,17 +553,31 @@ it('will redact by default if the config is set to do so', () => { [new URL(MOCK_DATA_SERVER).hostname]: { '/posts': { location: 'path', + method: 'GET', regex: '/posts', ignored: false, sensitiveKeys: [ - { keyPath: 'responseBody.user.email', action: SensitiveKeyActions.ALLOW }, - { keyPath: 'requestBody.blogType.name', action: SensitiveKeyActions.REDACT }, - { keyPath: 'responseBody.comments[].id', action: SensitiveKeyActions.ALLOW } + { + keyPath: 'responseBody.user.email', + action: SensitiveKeyActions.ALLOW + }, + { + keyPath: 'requestBody.blogType.name', + action: SensitiveKeyActions.REDACT + }, + { + keyPath: 'responseBody.comments[].id', + action: SensitiveKeyActions.ALLOW + } ] } } }; - const config = { remoteConfig, ...defaultConfig, redactByDefault: true } as ConfigType; + const config = { + remoteConfig, + ...defaultConfig, + redactByDefault: true + } as ConfigType; const events = prepareData([obj], config); expect(_get(events[0], 'request.body.blogType.name')).toBeFalsy(); expect(_get(events[0], 'response.body.name')).toBeFalsy(); @@ -562,16 +627,27 @@ it('will redact by default for an array of strings', () => { '/posts': { location: 'path', regex: '/posts', + method: 'GET', ignored: false, sensitiveKeys: [ - { keyPath: 'responseBody.user.email', action: SensitiveKeyActions.ALLOW }, - { keyPath: 'requestBody.blogType.name', action: SensitiveKeyActions.REDACT }, + { + keyPath: 'responseBody.user.email', + action: SensitiveKeyActions.ALLOW + }, + { + keyPath: 'requestBody.blogType.name', + action: SensitiveKeyActions.REDACT + }, { keyPath: 'responseBody.tags[]', action: SensitiveKeyActions.ALLOW } ] } } }; - const config = { remoteConfig, ...defaultConfig, redactByDefault: true } as ConfigType; + const config = { + remoteConfig, + ...defaultConfig, + redactByDefault: true + } as ConfigType; const events = prepareData([obj], config); expect(_get(events[0], 'request.body.blogType.name')).toBeFalsy(); expect(_get(events[0], 'response.body.name')).toBeFalsy(); @@ -611,7 +687,10 @@ it('will redact ONLY sensitive keys marked as redact, without either option enab name: 'John Doe', email: 'john@doe.com' }, - comments: [{ id: 7, comment: 'good blog'}, { id: 8, comment: 'bad blog'}] + comments: [ + { id: 7, comment: 'good blog' }, + { id: 8, comment: 'bad blog' } + ] } } }; @@ -620,11 +699,21 @@ it('will redact ONLY sensitive keys marked as redact, without either option enab '/posts': { location: 'path', regex: '/posts', + method: 'GET', ignored: false, sensitiveKeys: [ - { keyPath: 'responseBody.user.email', action: SensitiveKeyActions.ALLOW }, - { keyPath: 'requestBody.blogType.name', action: SensitiveKeyActions.REDACT }, - { keyPath: 'responseBody.comments[].id', action: SensitiveKeyActions.ALLOW } + { + keyPath: 'responseBody.user.email', + action: SensitiveKeyActions.ALLOW + }, + { + keyPath: 'requestBody.blogType.name', + action: SensitiveKeyActions.REDACT + }, + { + keyPath: 'responseBody.comments[].id', + action: SensitiveKeyActions.ALLOW + } ] } } diff --git a/src/utils.ts b/src/utils.ts index f5b898f..9447203 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -88,7 +88,7 @@ const getHeaderOptions = ( clientId + ':' + clientSecret ).toString('base64')}` }, - timeout, + timeout }; }; @@ -119,9 +119,11 @@ const unmarshalKeyPath = (keypath: string) => { const expandSensitiveKeySetForArrays = ( obj: any, sensitiveKeys: Array<{ keyPath: string; action: string }> -): Array<{ keyPath: string, action: string }> => { - - const expandKey = (key: { keyPath: string; action: string }, obj: any): Array<{ keyPath: string; action: string }> => { +): Array<{ keyPath: string; action: string }> => { + const expandKey = ( + key: { keyPath: string; action: string }, + obj: any + ): Array<{ keyPath: string; action: string }> => { // Split the key by dots, considering the array brackets as part of the key const parts = key?.keyPath.match(/[^.\[\]]+|\[\d*\]|\[\*\]/g) || []; // Recursively expand the key @@ -133,7 +135,6 @@ const expandSensitiveKeySetForArrays = ( obj: any, key: { keyPath: string; action: string } ): Array<{ keyPath: string; action: string }> => { - const path = key.keyPath; if (parts.length === 0) { return [{ keyPath: path, action: key.action }]; // Remove trailing dot @@ -150,22 +151,29 @@ const expandSensitiveKeySetForArrays = ( } // Expand for each element in the array return obj.flatMap((_, index) => - expand(parts.slice(1), obj[index], { keyPath: `${path}${separator}[${index}]`, action: key.action }) + expand(parts.slice(1), obj[index], { + keyPath: `${path}${separator}[${index}]`, + action: key.action + }) ); - } else if (part.startsWith('[') && part.endsWith(']')) { // Specific index in the array const index = parseInt(part.slice(1, -1), 10); if (!isNaN(index) && index < obj.length) { - return expand(parts.slice(1), obj[index], { keyPath: `${path}${separator}${part}`, action: key.action }); + return expand(parts.slice(1), obj[index], { + keyPath: `${path}${separator}${part}`, + action: key.action + }); } else { return []; } - } else { // Regular object property if (obj && typeof obj === 'object' && part in obj) { - return expand(parts.slice(1), obj[part], { keyPath: `${path}${separator}${part}`, action: key.action }); + return expand(parts.slice(1), obj[part], { + keyPath: `${path}${separator}${part}`, + action: key.action + }); } else { return []; } @@ -180,9 +188,11 @@ function getKeyPaths(obj: any, path: string = ''): string[] { if (typeof obj === 'object' && obj !== null) { // Object.keys returns indices for arrays - Object.keys(obj).forEach(key => { + Object.keys(obj).forEach((key) => { const value = obj[key]; - const newPath = Array.isArray(obj) ? `${path}[${key}]` : `${path}${path ? '.' : ''}${key}`; + const newPath = Array.isArray(obj) + ? `${path}[${key}]` + : `${path}${path ? '.' : ''}${key}`; if (typeof value === 'object' && value !== null) { paths = paths.concat(getKeyPaths(value, newPath)); } else { @@ -196,15 +206,19 @@ function getKeyPaths(obj: any, path: string = ''): string[] { return paths; } -const getAllKeyPathsForLeavesOnEvent = (event: { request?: RequestType; response?: ResponseType }) => [ - ...getKeyPaths(event.request?.headers, 'request.headers'), - ...getKeyPaths(event.request?.body, 'request.body'), - ...getKeyPaths(event.response?.headers, 'response.headers'), - ...getKeyPaths(event.response?.body, 'response.body') -].map((key) => ({ keyPath: key, action: SensitiveKeyActions.REDACT })); +const getAllKeyPathsForLeavesOnEvent = (event: { + request?: RequestType; + response?: ResponseType; +}) => + [ + ...getKeyPaths(event.request?.headers, 'request.headers'), + ...getKeyPaths(event.request?.body, 'request.body'), + ...getKeyPaths(event.response?.headers, 'response.headers'), + ...getKeyPaths(event.response?.body, 'response.body') + ].map((key) => ({ keyPath: key, action: SensitiveKeyActions.REDACT })); const redactValuesFromKeys = ( - event: { request?: RequestType; response?: ResponseType, tags?: TagType }, + event: { request?: RequestType; response?: ResponseType; tags?: TagType }, config: ConfigType ): { event: { request?: RequestType; response?: ResponseType }; @@ -212,10 +226,10 @@ const redactValuesFromKeys = ( tags: TagType; } => { const { redactByDefault, forceRedactAll } = config; - const remoteConfig = config?.remoteConfig || {} as RemoteConfigType; + const remoteConfig = config?.remoteConfig || ({} as RemoteConfigType); // Move the tags off the event object and into the metadata object let tags = {}; - if(event.tags) { + if (event.tags) { tags = event.tags; delete event.tags; } @@ -226,14 +240,19 @@ const redactValuesFromKeys = ( remoteConfig ); - if ((!endpointConfig || !endpointConfig?.sensitiveKeys?.length) && (!redactByDefault && !forceRedactAll)) { + if ( + (!endpointConfig || !endpointConfig?.sensitiveKeys?.length) && + !redactByDefault && + !forceRedactAll + ) { return { event, sensitiveKeyMetadata, tags }; } else { - - let sensitiveKeys = expandSensitiveKeySetForArrays( event, - (endpointConfig?.sensitiveKeys || []).map((key) => ({ keyPath: marshalKeyPath(key.keyPath), action: key.action })) + (endpointConfig?.sensitiveKeys || []).map((key) => ({ + keyPath: marshalKeyPath(key.keyPath), + action: key.action + })) ); if (forceRedactAll) { @@ -242,10 +261,17 @@ const redactValuesFromKeys = ( } else if (redactByDefault) { // Sensitive keys = All of the leaves on the event EXCEPT the ones marked allwoed from the remote config sensitiveKeys = (getAllKeyPathsForLeavesOnEvent(event) || []).filter( - (key) => !sensitiveKeys.some(sk => sk.keyPath === key.keyPath && sk.action === SensitiveKeyActions.ALLOW) + (key) => + !sensitiveKeys.some( + (sk) => + sk.keyPath === key.keyPath && + sk.action === SensitiveKeyActions.ALLOW + ) ); } else { - sensitiveKeys = sensitiveKeys.filter((sk) => sk.action !== SensitiveKeyActions.ALLOW); + sensitiveKeys = sensitiveKeys.filter( + (sk) => sk.action !== SensitiveKeyActions.ALLOW + ); } for (let i = 0; i < sensitiveKeys.length; i++) { @@ -268,11 +294,11 @@ const redactValuesFromKeys = ( const partition = (s: string, seperator: string) => { const index = s.indexOf(seperator); if (index === -1) { - return [s, '', ''] + return [s, '', '']; } else { - return [s.slice(0, index), seperator, s.slice(index+seperator.length)]; + return [s.slice(0, index), seperator, s.slice(index + seperator.length)]; } -} +}; const parseOneAsSSE = (chunk: string) => { // IF chunk is a single valid SSE event, returns chunk as an SSE object @@ -291,10 +317,10 @@ const parseOneAsSSE = (chunk: string) => { id, data, retry - } + }; } // otherwise keep building SSE - if (line.startsWith(":")) { + if (line.startsWith(':')) { // per SSE spec, this is invalid. return null; } @@ -305,7 +331,7 @@ const parseOneAsSSE = (chunk: string) => { } value = value.trimStart(); - switch(fieldName) { + switch (fieldName) { case 'event': event = value; break; @@ -313,7 +339,7 @@ const parseOneAsSSE = (chunk: string) => { data.push(safeParseJson(value)); break; case 'id': - if (!value.includes("\0")) { + if (!value.includes('\0')) { id = value; } break; @@ -324,7 +350,7 @@ const parseOneAsSSE = (chunk: string) => { } // No dispatch instruction, currently not a valid SSE return null; -} +}; const parseAsSSE = (stream: string) => { // If `stream` is a valid stream of server side events, @@ -338,22 +364,28 @@ const parseAsSSE = (stream: string) => { } let data = ''; - const responseBody = splits.map((split) => { - data += split; - if (data.endsWith('\n\n') || data.endsWith('\r\r') || data.endsWith('\r\n\r\n')) { - // Check if data is a valid SSE - const sse = parseOneAsSSE(data) - if (sse) { - // reset data to start building the next SSE - data = ''; - return sse; + const responseBody = splits + .map((split) => { + data += split; + if ( + data.endsWith('\n\n') || + data.endsWith('\r\r') || + data.endsWith('\r\n\r\n') + ) { + // Check if data is a valid SSE + const sse = parseOneAsSSE(data); + if (sse) { + // reset data to start building the next SSE + data = ''; + return sse; + } } - } - }).filter((sse) => !!sse); + }) + .filter((sse) => !!sse); // if there were no valid server sent events, return the original string // otherwise, return an array of SSE return responseBody?.length ? responseBody : null; -} +}; const safeParseJson = (json: string) => { try { @@ -372,17 +404,15 @@ const safeParseInt = (int: string) => { } finally { return null; } -} +}; const parseResponseBody = (rawResponseBody: string, contentType?: string) => { - - if(contentType?.includes(ContentType.EventStream)) { + if (contentType?.includes(ContentType.EventStream)) { return parseAsSSE(rawResponseBody); } else { return safeParseJson(rawResponseBody); } -} - +}; const redactValue = ( input: string | Record | [Record] | undefined @@ -414,7 +444,7 @@ const redactValue = ( const prepareData = ( events: Array, - supergoodConfig: ConfigType, + supergoodConfig: ConfigType ) => { return events.map((e) => { const { event, sensitiveKeyMetadata, tags } = redactValuesFromKeys( @@ -487,7 +517,11 @@ const post = ( }); }; -const get = (url: string, authorization: string, timeout: number): Promise => { +const get = ( + url: string, + authorization: string, + timeout: number +): Promise => { const packageVersion = version; const options = { @@ -538,14 +572,18 @@ const processRemoteConfig = (remoteConfigPayload: RemoteConfigPayloadType) => { return (remoteConfigPayload || []).reduce((remoteConfig, domainConfig) => { const { domain, endpoints } = domainConfig; const endpointConfig = endpoints.reduce((endpointConfig, endpoint) => { - const { matchingRegex, endpointConfiguration } = endpoint; + const { matchingRegex, endpointConfiguration, method } = endpoint; const { regex, location } = matchingRegex; const { action, sensitiveKeys } = endpointConfiguration; endpointConfig[regex] = { location, regex, + method, ignored: action === EndpointActions.IGNORE, - sensitiveKeys: (sensitiveKeys || []).map((key) => ({ keyPath: key.keyPath, action: key.action })) + sensitiveKeys: (sensitiveKeys || []).map((key) => ({ + keyPath: key.keyPath, + action: key.action + })) }; return endpointConfig; }, {} as { [endpointName: string]: EndpointConfigType }); @@ -584,7 +622,10 @@ const getEndpointConfigForRequest = ( for (let i = 0; i < Object.keys(endpointConfigs).length; i++) { const endpointConfig = endpointConfigs[Object.keys(endpointConfigs)[i]]; - const { regex, location } = endpointConfig; + const { regex, location, method } = endpointConfig; + if (request.method.toLocaleLowerCase() !== method.toLocaleLowerCase()) { + continue; + } const regexObj = new RegExp(regex); const strRepresentation = getStrRepresentationFromPath(request, location); if (!strRepresentation) continue; diff --git a/test/e2e/remote-config.e2e.test.ts b/test/e2e/remote-config.e2e.test.ts index a099acf..5ecf938 100644 --- a/test/e2e/remote-config.e2e.test.ts +++ b/test/e2e/remote-config.e2e.test.ts @@ -37,6 +37,7 @@ describe('remote config functionality', () => { endpoints: [ { name: '/posts', + method: 'GET', matchingRegex: { regex: '/posts', location: 'path' @@ -72,6 +73,7 @@ describe('remote config functionality', () => { endpoints: [ { name: '/profile', + method: 'GET', matchingRegex: { regex: '/profile', location: 'path' @@ -111,6 +113,7 @@ describe('remote config functionality', () => { endpoints: [ { name: '/posts', + method: 'GET', matchingRegex: { regex: '/posts', location: 'path'