diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c657037..61b4027c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -125,7 +125,7 @@ jobs: environment: - TZ: Asia/Jerusalem - NODE_OPTIONS: --max_old_space_size=1500 - resource_class: medium+ + resource_class: large working_directory: ~/lumigo-node steps: - run: @@ -150,13 +150,13 @@ jobs: - run: name: check types command: npm run check-types + - run: + name: eslint + command: npm run lint - run: name: test command: npm test no_output_timeout: 15m - - run: - name: eslint - command: npm run lint deploy: <<: *defaults diff --git a/src/utils.test.js b/src/utils.test.js index 39fda672..f036fd40 100644 --- a/src/utils.test.js +++ b/src/utils.test.js @@ -34,6 +34,7 @@ import { LUMIGO_SUPPORT_LARGE_INVOCATIONS, removeLumigoFromError, removeLumigoFromStacktrace, + LUMIGO_SECRET_MASKING_DEBUG, } from './utils'; describe('utils', () => { @@ -248,6 +249,12 @@ describe('utils', () => { expect(utils.isDebug()).toBe(true); }); + test('isSecretMaskingDebug -> ENV VAR', () => { + expect(utils.isSecretMaskingDebug()).toBe(false); + process.env.LUMIGO_SECRET_MASKING_DEBUG = 'TRUE'; + expect(utils.isSecretMaskingDebug()).toBe(true); + }); + test('isLambdaWrapped', () => { expect(utils.isLambdaWrapped()).toBe(false); process.env.LUMIGO_IS_WRAPPED = 'TRUE'; diff --git a/src/utils.ts b/src/utils.ts index 8881d38f..bd2cc65a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -33,6 +33,8 @@ export const LUMIGO_SECRET_MASKING_REGEX_HTTP_QUERY_PARAMS = export const LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT = 'LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT'; export const LUMIGO_SECRET_MASKING_ALL_MAGIC = 'all'; +export const LUMIGO_SECRET_MASKING_DEBUG = 'LUMIGO_SECRET_MASKING_DEBUG'; + export const LUMIGO_SECRET_MASKING_EXACT_PATH = 'LUMIGO_SECRET_MASKING_EXACT_PATH'; export const LUMIGO_WHITELIST_KEYS_REGEXES = 'LUMIGO_WHITELIST_KEYS_REGEXES'; export const LUMIGO_SUPPORT_LARGE_INVOCATIONS = 'LUMIGO_SUPPORT_LARGE_INVOCATIONS'; @@ -50,6 +52,18 @@ export const OMITTING_KEYS_REGEXES = [ 'Authorization', ]; +export const BYPASS_MASKING_KEYS = [ + LUMIGO_SECRET_MASKING_REGEX, + LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP, + LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_BODIES, + LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_HEADERS, + LUMIGO_SECRET_MASKING_REGEX_HTTP_RESPONSE_BODIES, + LUMIGO_SECRET_MASKING_REGEX_HTTP_RESPONSE_HEADERS, + LUMIGO_SECRET_MASKING_REGEX_ENVIRONMENT, + LUMIGO_SECRET_MASKING_REGEX_HTTP_QUERY_PARAMS, + LUMIGO_SECRET_MASKING_EXACT_PATH, +]; + export const LUMIGO_EVENT_KEY = '_lumigo'; export const STEP_FUNCTION_UID_KEY = 'step_function_uid'; export const GET_KEY_DEPTH_ENV_KEY = 'LUMIGO_KEY_DEPTH'; @@ -339,6 +353,8 @@ export const isWarm = (): boolean => export const isDebug = (): boolean => validateEnvVar(DEBUG_FLAG) || TracerGlobals.getTracerInputs().debug; +export const isSecretMaskingDebug = (): boolean => validateEnvVar(LUMIGO_SECRET_MASKING_DEBUG); + export const isLambdaWrapped = (): boolean => validateEnvVar(WRAPPED_FLAG); export const shouldPropagateW3C = (): boolean => !validateEnvVar(LUMIGO_PROPAGATE_W3C, 'FALSE'); @@ -423,6 +439,8 @@ export const setSwitchOff = () => (process.env['LUMIGO_SWITCH_OFF'] = 'TRUE'); export const setDebug = () => (process.env['LUMIGO_DEBUG'] = 'TRUE'); +export const setSecretMaskingDebug = () => (process.env['LUMIGO_SECRET_MASKING_DEBUG'] = 'TRUE'); + export const unsetDebug = () => (process.env['LUMIGO_DEBUG'] = undefined); export const setTimeoutTimerDisabled = () => (process.env[TIMEOUT_ENABLE_FLAG] = 'FALSE'); diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index 7e71959f..61ac8bfb 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -17,6 +17,8 @@ import { OMITTING_KEYS_REGEXES, parseJsonFromEnvVar, safeExecute, + BYPASS_MASKING_KEYS, + isSecretMaskingDebug, } from '../utils'; import { runOneTimeWrapper } from './functionUtils'; @@ -32,6 +34,11 @@ const keyToRegexes = ( backwardCompRegexEnvVarName = LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP, regexesEnvVarName = LUMIGO_SECRET_MASKING_REGEX ) => { + logSecretMaskingDebug(logger, 'Getting key to omit regexes', { + regexesList, + backwardCompRegexEnvVarName, + regexesEnvVarName, + }); const fallbackRegexesList = regexesList; const tryParseEnvVar = (envVarName) => { @@ -162,6 +169,16 @@ function innerPathScrubbing(input, secretPaths, uniquePaths, currentPath) { return input; } +function logSecretMaskingDebug(logger, message, additionalData) { + if (isSecretMaskingDebug()) { + if (additionalData) { + logger.debug(message, additionalData); + } else { + logger.debug(message); + } + } +} + export const payloadStringify = ( payload, maxPayloadSize = getEventEntitySize(), @@ -258,8 +275,13 @@ const invalidMaskingRegexWarning = runOneTimeWrapper((e) => { }); const shallowMaskByRegex = (payload, regexes) => { + logSecretMaskingDebug(logger, 'Shallow masking payload by regexes', { + payloadKeys: Object.keys(payload), + regexes, + }); regexes = regexes || keyToOmitRegexes(); if (isString(payload)) { + logSecretMaskingDebug(logger, 'Shallow masking string payload'); return payload; } if (typeof payload !== 'object') { @@ -267,7 +289,11 @@ const shallowMaskByRegex = (payload, regexes) => { return payload; } return Object.keys(payload).reduce((acc, key) => { - if (keyContainsRegex(regexes, key)) { + if (BYPASS_MASKING_KEYS.includes(key)) { + logSecretMaskingDebug(logger, 'Skipping masking of a Lumigo env-var', key); + acc[key] = payload[key]; + } else if (keyContainsRegex(regexes, key)) { + logSecretMaskingDebug(logger, 'Shallow masking key', key); acc[key] = SCRUBBED_TEXT; } else { acc[key] = payload[key]; @@ -277,6 +303,10 @@ const shallowMaskByRegex = (payload, regexes) => { }; export const shallowMask = (context, payload) => { + logSecretMaskingDebug(logger, 'Shallow masking payload', { + context, + payloadKeys: Object.keys(payload), + }); let givenSecretRegexes = null; if (context === 'environment') { givenSecretRegexes = getEnvVarsMaskingRegex(); @@ -295,10 +325,15 @@ export const shallowMask = (context, payload) => { } if (givenSecretRegexes === LUMIGO_SECRET_MASKING_ALL_MAGIC) { + logSecretMaskingDebug(logger, 'Shallow masking payload with LUMIGO_SECRET_MASKING_ALL_MAGIC'); return SCRUBBED_TEXT; } else if (givenSecretRegexes) { + logSecretMaskingDebug(logger, 'Shallow masking payload with given regexes', { + givenSecretRegexes, + }); try { givenSecretRegexes = JSON.parse(givenSecretRegexes); + logSecretMaskingDebug(logger, 'Parsed given regexes', { givenSecretRegexes }); givenSecretRegexes = givenSecretRegexes.map((x) => new RegExp(x, 'i')); } catch (e) { invalidMaskingRegexWarning(e); diff --git a/src/utils/payloadStringify.test.js b/src/utils/payloadStringify.test.js index 3c49f833..0ddf59e3 100644 --- a/src/utils/payloadStringify.test.js +++ b/src/utils/payloadStringify.test.js @@ -3,6 +3,7 @@ import { LUMIGO_SECRET_MASKING_ALL_MAGIC, LUMIGO_SECRET_MASKING_EXACT_PATH, LUMIGO_SECRET_MASKING_REGEX, + OMITTING_KEYS_REGEXES, LUMIGO_SECRET_MASKING_REGEX_BACKWARD_COMP, LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_BODIES, LUMIGO_WHITELIST_KEYS_REGEXES, @@ -298,6 +299,23 @@ describe('payloadStringify', () => { expect(shallowMask('requestBody', { a: 'b', aXy: 'bla' })).toEqual({ a: 'b', aXy: '****' }); }); + test('shallowMask -> requestBody -> regex -> bypass', () => { + const regex = '[".*X.*"]'; + process.env[LUMIGO_SECRET_MASKING_REGEX] = regex; + + expect( + shallowMask('environment', { + LUMIGO_SECRET_MASKING_REGEX: regex, + a: 'b', + aXy: 'some secret', + }) + ).toEqual({ + LUMIGO_SECRET_MASKING_REGEX: regex, + a: 'b', + aXy: '****', + }); + }); + test('shallowMask -> requestBody -> fallback', () => { expect(shallowMask('requestBody', { a: 'b', password: 'bla' })).toEqual({ a: 'b', @@ -313,7 +331,13 @@ describe('payloadStringify', () => { utils.setDebug(); TracerGlobals.setTracerInputs({}); expect(shallowMask('requestBody', 1)).toEqual(1); - expect(ConsoleWritesForTesting.getLogs()).toEqual([ + + // Filter logs to only include WARNING logs + const warningLogs = ConsoleWritesForTesting.getLogs().filter((log) => + log.msg.includes('WARNING') + ); + + expect(warningLogs).toEqual([ { msg: '#LUMIGO# - WARNING - "Failed to mask payload, payload is not an object or string"', obj: '1', @@ -325,11 +349,16 @@ describe('payloadStringify', () => { utils.setDebug(); TracerGlobals.setTracerInputs({}); expect(shallowMask('other', { a: 'b', password: 1234 })).toEqual({ a: 'b', password: '****' }); - expect(ConsoleWritesForTesting.getLogs()).toEqual([ - { + + // Filter logs to only include WARNING logs + const warningLogs = ConsoleWritesForTesting.getLogs().filter((log) => + log.msg.includes('WARNING') + ); + + expect(warningLogs).toEqual([ + expect.objectContaining({ msg: '#LUMIGO# - WARNING - "Unknown context for shallowMask"', - obj: '"other"', - }, + }), ]); }); @@ -338,7 +367,12 @@ describe('payloadStringify', () => { process.env[LUMIGO_SECRET_MASKING_REGEX] = '["a(a"]'; expect(shallowMask('requestBody', { a: 'b', aa: 'bla' })).toEqual({ a: 'b', aa: 'bla' }); - expect(ConsoleWritesForTesting.getLogs()).toEqual([ + // Filter logs to only include WARNING logs + const warningLogs = ConsoleWritesForTesting.getLogs().filter((log) => + log.msg.includes('WARNING') + ); + + expect(warningLogs).toEqual([ expect.objectContaining({ msg: '#LUMIGO# - WARNING - "Failed to parse the given masking regex"', }), @@ -348,6 +382,23 @@ describe('payloadStringify', () => { ]); }); + test('shallowMask -> LUMIGO_SECRET_MASKING_DEBUG', () => { + utils.setDebug(); + utils.setSecretMaskingDebug(); + + expect(shallowMask('requestBody', { a: 'b' })).toEqual({ a: 'b' }); + + const debugLogs = ConsoleWritesForTesting.getLogs().filter((log) => log.msg.includes('DEBUG')); + + expect(debugLogs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + msg: '#LUMIGO# - DEBUG - "Shallow masking payload"', + }), + ]) + ); + }); + test.each` envVarValue | event | expectedResults ${['["object.foo"]']} | ${[{ secret: { key: 'value' } }, { object: { foo: 'value' } }]} | ${JSON.stringify([{ secret: '****' }, { object: { foo: '****' } }])}