From 5b23d8275dba7a2d4b19cd21bfa394ebca392c3e Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Tue, 9 Jul 2024 17:32:05 +0300 Subject: [PATCH 01/14] fix: rewrite the max latency reached log --- src/utils/globalDurationTimer.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/utils/globalDurationTimer.ts b/src/utils/globalDurationTimer.ts index fedbaba7a..afa4d2b9c 100644 --- a/src/utils/globalDurationTimer.ts +++ b/src/utils/globalDurationTimer.ts @@ -1,9 +1,13 @@ import * as logger from '../logger'; -import { getTracerMaxDurationTimeout } from '../utils'; +import { getTracerMaxDurationTimeout, TRACER_TIMEOUT_FLAG } from '../utils'; import { runOneTimeWrapper } from './functionUtils'; -const warnTimeoutOnce = runOneTimeWrapper(() => { - logger.warn('Lumigo tracer is no longer collecting data on the invocation.'); +const warnTimeoutOnce = runOneTimeWrapper((threshold: number, currentDuration: number) => { + logger.info( + `Lumigo tracer is no longer collecting data on the invocation, because it reached the maximum + of ${threshold} MS added (added ${currentDuration} MS so far) to the lambda duration. + This limit can be modified by setting the ${TRACER_TIMEOUT_FLAG} environment variable` + ); }, {}); export const GlobalDurationTimer = (() => { @@ -32,7 +36,7 @@ export const GlobalDurationTimer = (() => { appendTime(); threshold = threshold || getTracerMaxDurationTimeout(); if (currentDuration >= threshold) { - warnTimeoutOnce(); + warnTimeoutOnce(threshold, currentDuration); return true; } return false; From 36003c65fe695c9eb06033db3cc5381136d4d9e8 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Tue, 9 Jul 2024 17:34:04 +0300 Subject: [PATCH 02/14] fix: debug log when span skipped due to size limitation --- src/globals.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/globals.ts b/src/globals.ts index 4c89d98bc..11bc4d030 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -5,7 +5,6 @@ import { BasicSpan } from './types/spans/basicSpan'; import { getAutoTagKeys, getJSONBase64Size, - getMaxRequestSize, getMaxRequestSizeOnError, isLambdaTraced, spanHasErrors, @@ -39,6 +38,10 @@ export class SpansContainer { logger.debug('Span created', span); return true; } + + logger.debug('Span was not added due to size limitations', { + currentSpansSize: this.currentSpansSize, + }); return false; } From 3de02595ba072915f1ba4921045d41a82d309386 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Tue, 9 Jul 2024 17:35:09 +0300 Subject: [PATCH 03/14] fix: better debug log for splitting spans into zip bulks --- src/reporter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/reporter.ts b/src/reporter.ts index 65339d692..bd0b514ff 100644 --- a/src/reporter.ts +++ b/src/reporter.ts @@ -190,7 +190,7 @@ export const forgeAndScrubRequestBody = ( ) { if (shouldTryZip) { logger.debug( - `Spans are too big, size [${size}], bigger than: [${maxRequestSize}], trying to split and zip` + `Spans are too big, size [${size}], [${spans.length}] spans, bigger than: [${maxRequestSize}], trying to split and zip` ); const zippedSpansBulks = splitAndZipSpans(spans); const areAllSpansSmallEnough = zippedSpansBulks.every( From 07e50e2c4d94b1579a7236017a7cb3971d7186f8 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Tue, 9 Jul 2024 17:35:35 +0300 Subject: [PATCH 04/14] fix: missing debug log info --- src/tracer/tracer.ts | 2 +- src/utils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tracer/tracer.ts b/src/tracer/tracer.ts index a6449ace5..5630ee31b 100644 --- a/src/tracer/tracer.ts +++ b/src/tracer/tracer.ts @@ -159,7 +159,7 @@ export const sendEndTraceSpans = async (functionSpan, handlerReturnValue) => { logLeakedSpans(spans); const { transactionId } = endFunctionSpan; - logger.debug('Tracer ended', { transactionId }); + logger.debug('Tracer ended', { transactionId, totalSpans: spans.length }); clearGlobals(); }; diff --git a/src/utils.ts b/src/utils.ts index 96185ec62..17e7ae53f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -225,7 +225,6 @@ const STORE_LOGS_FLAG = 'LUMIGO_STORE_LOGS'; const TIMEOUT_BUFFER_FLAG = 'LUMIGO_TIMEOUT_BUFFER'; const TIMEOUT_MIN_DURATION = 'LUMIGO_TIMEOUT_MIN_DURATION'; const TIMEOUT_BUFFER_FLAG_MS = 'LUMIGO_TIMEOUT_BUFFER_MS'; -const TRACER_TIMEOUT_FLAG = 'LUMIGO_TRACER_TIMEOUT'; const AGENT_KEEPALIVE = 'LUMIGO_AGENT_KEEPALIVE_MS'; const REUSE_CONNECTION = 'LUMIGO_REUSE_HTTP_CONNECTION'; const KEEP_HEADERS = 'LUMIGO_KEEP_HTTP_HEADERS'; @@ -240,6 +239,7 @@ const LUMIGO_STACK_PATTERNS = [ new RegExp('/node_modules/@lumigo/tracer/', 'i'), ]; +export const TRACER_TIMEOUT_FLAG = 'LUMIGO_TRACER_TIMEOUT'; export const SWITCH_OFF_FLAG = 'LUMIGO_SWITCH_OFF'; const validateEnvVar = (envVar: string, value: string = 'TRUE'): boolean => From 39d05be242c9e6106f833876f7aa0755993ff635 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Tue, 9 Jul 2024 17:56:04 +0300 Subject: [PATCH 05/14] fix: log when spans are skipped due to large total spans size --- src/globals.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/globals.ts b/src/globals.ts index 11bc4d030..6d0e23b38 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -11,6 +11,7 @@ import { } from './utils'; import { GlobalDurationTimer } from './utils/globalDurationTimer'; import { isString } from '@lumigo/node-core/lib/common'; +import { runOneTimeWrapper } from './utils/functionUtils'; const MAX_TAGS = 50; const MAX_TAG_KEY_LEN = 50; @@ -21,6 +22,13 @@ export const DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR = 1024 * 990; export const MAX_TRACER_ADDED_DURATION_ALLOWED = 750; export const MIN_TRACER_ADDED_DURATION_ALLOWED = 200; +const warnSpansSizeOnce = runOneTimeWrapper((threshold: number, currentSize: number) => { + logger.info( + `Lumigo tracer is no longer collecting data on the invocation, because it reached the maximum + size of calls collected of ${threshold} bytes (current size is ${currentSize} bytes)` + ); +}, {}); + export class SpansContainer { private static spans: { [id: string]: BasicSpan } = {}; private static currentSpansSize: number = 0; @@ -39,6 +47,7 @@ export class SpansContainer { return true; } + warnSpansSizeOnce(getMaxRequestSizeOnError() * 10, this.currentSpansSize); logger.debug('Span was not added due to size limitations', { currentSpansSize: this.currentSpansSize, }); From 71aeb398036cf4bd38ab66a0db40e75f6a9f5472 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Wed, 10 Jul 2024 11:31:05 +0300 Subject: [PATCH 06/14] fix: change timeout log msg content --- src/utils/globalDurationTimer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/globalDurationTimer.ts b/src/utils/globalDurationTimer.ts index afa4d2b9c..68854b91a 100644 --- a/src/utils/globalDurationTimer.ts +++ b/src/utils/globalDurationTimer.ts @@ -4,8 +4,8 @@ import { runOneTimeWrapper } from './functionUtils'; const warnTimeoutOnce = runOneTimeWrapper((threshold: number, currentDuration: number) => { logger.info( - `Lumigo tracer is no longer collecting data on the invocation, because it reached the maximum - of ${threshold} MS added (added ${currentDuration} MS so far) to the lambda duration. + `Stopped collecting data for this invocation because it reached the maximum + of ${threshold} ms added (added ${currentDuration} ms so far) to the duration of the Lambda. This limit can be modified by setting the ${TRACER_TIMEOUT_FLAG} environment variable` ); }, {}); From 71e0539866bfdc4542b3bee638a52c4be7ffff05 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Wed, 10 Jul 2024 12:32:02 +0300 Subject: [PATCH 07/14] feat: env var for max spans size stored --- src/globals.ts | 7 ++++--- src/utils.ts | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/globals.ts b/src/globals.ts index 6d0e23b38..beebacbcf 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -5,7 +5,7 @@ import { BasicSpan } from './types/spans/basicSpan'; import { getAutoTagKeys, getJSONBase64Size, - getMaxRequestSizeOnError, + getStoredSpansMaxSize, isLambdaTraced, spanHasErrors, } from './utils'; @@ -40,17 +40,18 @@ export class SpansContainer { this.totalSpans += 1; } // Memory optimization, take up to 10x maxSize because of smart span selection logic - if (spanHasErrors(span) || getMaxRequestSizeOnError() * 10 > this.currentSpansSize) { + const maxSpansSize = getStoredSpansMaxSize(); + if (spanHasErrors(span) || maxSpansSize > this.currentSpansSize) { this.spans[span.id] = span; this.currentSpansSize += getJSONBase64Size(span); logger.debug('Span created', span); return true; } - warnSpansSizeOnce(getMaxRequestSizeOnError() * 10, this.currentSpansSize); logger.debug('Span was not added due to size limitations', { currentSpansSize: this.currentSpansSize, }); + warnSpansSizeOnce(maxSpansSize, this.currentSpansSize); return false; } diff --git a/src/utils.ts b/src/utils.ts index 17e7ae53f..be592c1a9 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -36,6 +36,7 @@ export const LUMIGO_SECRET_MASKING_ALL_MAGIC = 'all'; export const LUMIGO_SECRET_MASKING_EXACT_PATH = 'LUMIGO_SECRET_MASKING_EXACT_PATH'; export const LUMIGO_WHITELIST_KEYS_REGEXES = 'LUMIGO_WHITELIST_KEYS_REGEXES'; export const LUMIGO_SUPPORT_LARGE_INVOCATIONS = 'LUMIGO_SUPPORT_LARGE_INVOCATIONS'; +export const LUMIGO_STORED_SPANS_MAX_SIZE_BYTES_ENV_VAR = 'LUMIGO_STORED_SPANS_MAX_SIZE_BYTES'; export const OMITTING_KEYS_REGEXES = [ '.*pass.*', '.*key.*', @@ -378,6 +379,20 @@ export const getMaxRequestSize = () => TracerGlobals.getTracerInputs().maxSizeFo export const getMaxRequestSizeOnError = () => TracerGlobals.getTracerInputs().maxSizeForRequestOnError; +/** + * The maximum size of all spans stored in memory before sending them to lumigo. + * This limit is in place to prevent storing too many spans in memory and causing OOM errors. + * Note: when the invocation ends and the spans are processed before sending to Lumigo, more processing and truncating + * might take place + * @returns {number} The maximum size of all spans stored in memory in bytes + */ +export const getStoredSpansMaxSize = (): number => { + return ( + parseInt(process.env[LUMIGO_STORED_SPANS_MAX_SIZE_BYTES_ENV_VAR]) || + getMaxRequestSizeOnError() * 10 + ); +}; + export const getInvokedArn = () => { // @ts-ignore return TracerGlobals.getHandlerInputs().context.invokedFunctionArn || ''; From 633cd8873afbf94be394b38eb6ba6f2e2cec416a Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Wed, 10 Jul 2024 13:49:19 +0300 Subject: [PATCH 08/14] fix: line break in log msg --- src/globals.ts | 4 ++-- src/utils/globalDurationTimer.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/globals.ts b/src/globals.ts index beebacbcf..266f9aa00 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -24,8 +24,8 @@ export const MIN_TRACER_ADDED_DURATION_ALLOWED = 200; const warnSpansSizeOnce = runOneTimeWrapper((threshold: number, currentSize: number) => { logger.info( - `Lumigo tracer is no longer collecting data on the invocation, because it reached the maximum - size of calls collected of ${threshold} bytes (current size is ${currentSize} bytes)` + `Lumigo tracer is no longer collecting data on the invocation, because it reached the maximum ` + + `size of calls collected of ${threshold} bytes (current size is ${currentSize} bytes)` ); }, {}); diff --git a/src/utils/globalDurationTimer.ts b/src/utils/globalDurationTimer.ts index 68854b91a..8f1a59077 100644 --- a/src/utils/globalDurationTimer.ts +++ b/src/utils/globalDurationTimer.ts @@ -4,9 +4,9 @@ import { runOneTimeWrapper } from './functionUtils'; const warnTimeoutOnce = runOneTimeWrapper((threshold: number, currentDuration: number) => { logger.info( - `Stopped collecting data for this invocation because it reached the maximum - of ${threshold} ms added (added ${currentDuration} ms so far) to the duration of the Lambda. - This limit can be modified by setting the ${TRACER_TIMEOUT_FLAG} environment variable` + `Stopped collecting data for this invocation because it reached the maximum ` + + `of ${threshold} ms added (added ${currentDuration} ms so far) to the duration of the Lambda. ` + + `This limit can be modified by setting the ${TRACER_TIMEOUT_FLAG} environment variable` ); }, {}); From fffd183ee8201f6863d91b89a07c9483cb9e6af1 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Wed, 10 Jul 2024 14:23:11 +0300 Subject: [PATCH 09/14] test: getStoredSpansMaxSize using value from env var if present --- src/globals.test.js | 2 +- src/utils.test.js | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/globals.test.js b/src/globals.test.js index e629b990a..df130b772 100644 --- a/src/globals.test.js +++ b/src/globals.test.js @@ -64,7 +64,7 @@ describe('globals', () => { expect(spans).toEqual([span1]); }); - test('SpansContainer - cleanning the request size limiter', () => { + test('SpansContainer - cleaning the request size limiter', () => { const span1 = { a: 'b', c: 'd', id: '1' }; for (let i = 0; i < getMaxRequestSize(); i++) { globals.SpansContainer.addSpan(span1); diff --git a/src/utils.test.js b/src/utils.test.js index 70b13b234..88805e410 100644 --- a/src/utils.test.js +++ b/src/utils.test.js @@ -29,6 +29,8 @@ import { shouldScrubDomain, getTraceId, safeExecuteAsync, + getStoredSpansMaxSize, + LUMIGO_STORED_SPANS_MAX_SIZE_BYTES_ENV_VAR, } from './utils'; describe('utils', () => { @@ -311,6 +313,12 @@ describe('utils', () => { expect(utils.getEventEntitySize()).toBe(DEFAULT_LUMIGO_MAX_ENTRY_SIZE); }); + test('getStoredSpansMaxSize using env var value', () => { + const valueBefore = getStoredSpansMaxSize(); + process.env[LUMIGO_STORED_SPANS_MAX_SIZE_BYTES_ENV_VAR] = `${valueBefore + 100}`; + expect(getStoredSpansMaxSize()).toBe(valueBefore + 100); + }); + test('getConnectionTimeout => simple flow', () => { process.env.LUMIGO_CONNECTION_TIMEOUT = '600'; expect(utils.getConnectionTimeout()).toBe(600); From 81a779e4e22d5627c89daec5a53ba8812d5fda77 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Wed, 10 Jul 2024 17:05:19 +0300 Subject: [PATCH 10/14] fix: rewrite loading max size of stored spans code --- src/globals.test.js | 6 ++++ src/globals.ts | 26 +++++++++++----- src/reporter.test.ts | 2 +- src/tracer/tracer-options.interface.ts | 1 + src/utils.test.js | 10 +++++-- src/utils.ts | 41 ++++++++++++++++---------- 6 files changed, 58 insertions(+), 28 deletions(-) diff --git a/src/globals.test.js b/src/globals.test.js index df130b772..97da0584a 100644 --- a/src/globals.test.js +++ b/src/globals.test.js @@ -3,6 +3,7 @@ import * as globals from './globals'; import { DEFAULT_MAX_SIZE_FOR_REQUEST, DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR, + DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, MAX_TRACER_ADDED_DURATION_ALLOWED, } from './globals'; import { getMaxRequestSize } from './utils'; @@ -238,6 +239,7 @@ describe('globals', () => { const isStepFunction = false; const maxSizeForRequest = 1234; const maxSizeForRequestOnError = 12345; + const maxSizeForStoredSpansInMemory = 1234567; globals.TracerGlobals.setTracerInputs({ token, debug, @@ -246,6 +248,7 @@ describe('globals', () => { isStepFunction, maxSizeForRequest, maxSizeForRequestOnError, + maxSizeForStoredSpansInMemory, }); expect(globals.TracerGlobals.getTracerInputs()).toEqual({ token, @@ -256,6 +259,7 @@ describe('globals', () => { lambdaTimeout: MAX_TRACER_ADDED_DURATION_ALLOWED, maxSizeForRequest, maxSizeForRequestOnError, + maxSizeForStoredSpansInMemory, }); globals.TracerGlobals.clearTracerInputs(); expect(globals.TracerGlobals.getTracerInputs()).toEqual({ @@ -267,6 +271,7 @@ describe('globals', () => { isStepFunction: false, maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST, maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR, + maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, }); }); @@ -310,6 +315,7 @@ describe('globals', () => { lambdaTimeout: MAX_TRACER_ADDED_DURATION_ALLOWED, maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST, maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR, + maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, }); }); diff --git a/src/globals.ts b/src/globals.ts index 266f9aa00..bbf340f00 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -5,7 +5,7 @@ import { BasicSpan } from './types/spans/basicSpan'; import { getAutoTagKeys, getJSONBase64Size, - getStoredSpansMaxSize, + getMaxSizeForStoredSpansInMemory, isLambdaTraced, spanHasErrors, } from './utils'; @@ -19,6 +19,8 @@ const MAX_TAG_VALUE_LEN = 70; const ADD_TAG_ERROR_MSG_PREFIX = 'Skipping addExecutionTag: Unable to add tag'; export const DEFAULT_MAX_SIZE_FOR_REQUEST = 1024 * 500; export const DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR = 1024 * 990; +export const DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY = + DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR * 10; export const MAX_TRACER_ADDED_DURATION_ALLOWED = 750; export const MIN_TRACER_ADDED_DURATION_ALLOWED = 200; @@ -40,7 +42,7 @@ export class SpansContainer { this.totalSpans += 1; } // Memory optimization, take up to 10x maxSize because of smart span selection logic - const maxSpansSize = getStoredSpansMaxSize(); + const maxSpansSize = getMaxSizeForStoredSpansInMemory(); if (spanHasErrors(span) || maxSpansSize > this.currentSpansSize) { this.spans[span.id] = span; this.currentSpansSize += getJSONBase64Size(span); @@ -226,6 +228,7 @@ export const TracerGlobals = (() => { isStepFunction: false, maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST, maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR, + maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, lambdaTimeout: MAX_TRACER_ADDED_DURATION_ALLOWED, }; @@ -254,7 +257,15 @@ export const TracerGlobals = (() => { maxSizeForRequest = null, maxSizeForRequestOnError = null, lambdaTimeout = MAX_TRACER_ADDED_DURATION_ALLOWED, - }: TracerOptions) => + maxSizeForStoredSpansInMemory = DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, + }: TracerOptions) => { + const parsedMaxSizeForRequestOnError = + maxSizeForRequestOnError || + (process.env['LUMIGO_MAX_SIZE_FOR_REQUEST_ON_ERROR'] + ? parseInt(process.env.LUMIGO_MAX_SIZE_FOR_REQUEST_ON_ERROR) + : DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR); + const parsedMaxSizeForStoredSpansInMemory = + maxSizeForStoredSpansInMemory || parsedMaxSizeForRequestOnError * 10; Object.assign(tracerInputs, { token: token || process.env.LUMIGO_TRACER_TOKEN, debug: debug, @@ -272,12 +283,10 @@ export const TracerGlobals = (() => { (process.env['LUMIGO_MAX_SIZE_FOR_REQUEST'] ? parseInt(process.env.LUMIGO_MAX_SIZE_FOR_REQUEST) : DEFAULT_MAX_SIZE_FOR_REQUEST), - maxSizeForRequestOnError: - maxSizeForRequestOnError || - (process.env['LUMIGO_MAX_SIZE_FOR_REQUEST_ON_ERROR'] - ? parseInt(process.env.LUMIGO_MAX_SIZE_FOR_REQUEST_ON_ERROR) - : DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR), + maxSizeForRequestOnError: parsedMaxSizeForRequestOnError, + maxSizeForStoredSpansInMemory: parsedMaxSizeForStoredSpansInMemory, }); + }; const getTracerInputs = () => tracerInputs; @@ -290,6 +299,7 @@ export const TracerGlobals = (() => { isStepFunction: false, maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST, maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR, + maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, }); return { diff --git a/src/reporter.test.ts b/src/reporter.test.ts index 169c86017..ba345b788 100644 --- a/src/reporter.test.ts +++ b/src/reporter.test.ts @@ -1148,7 +1148,7 @@ describe('reporter', () => { expect(JSON.parse(logs[1].obj).stack).toBeTruthy(); }); - test('sending a huge payload, bigger than the regular limit, zipping and geeting is back completely', async () => { + test('sending a huge payload, bigger than the regular limit, zipping and getting is back completely', async () => { const token = 'DEADBEEF'; TracerGlobals.setTracerInputs({ token }); const span = { diff --git a/src/tracer/tracer-options.interface.ts b/src/tracer/tracer-options.interface.ts index bd9b8e38c..f02f35b51 100644 --- a/src/tracer/tracer-options.interface.ts +++ b/src/tracer/tracer-options.interface.ts @@ -8,4 +8,5 @@ export interface TracerOptions { maxSizeForRequestOnError?: number | null; lambdaTimeout?: number; verbose?: boolean; + maxSizeForStoredSpansInMemory?: number; } diff --git a/src/utils.test.js b/src/utils.test.js index 88805e410..bb914bca2 100644 --- a/src/utils.test.js +++ b/src/utils.test.js @@ -29,8 +29,9 @@ import { shouldScrubDomain, getTraceId, safeExecuteAsync, - getStoredSpansMaxSize, + getMaxSizeForStoredSpansInMemory, LUMIGO_STORED_SPANS_MAX_SIZE_BYTES_ENV_VAR, + LUMIGO_SUPPORT_LARGE_INVOCATIONS, } from './utils'; describe('utils', () => { @@ -314,9 +315,12 @@ describe('utils', () => { }); test('getStoredSpansMaxSize using env var value', () => { - const valueBefore = getStoredSpansMaxSize(); + const valueBefore = getMaxSizeForStoredSpansInMemory(); process.env[LUMIGO_STORED_SPANS_MAX_SIZE_BYTES_ENV_VAR] = `${valueBefore + 100}`; - expect(getStoredSpansMaxSize()).toBe(valueBefore + 100); + expect(getMaxSizeForStoredSpansInMemory()).toBe(valueBefore); + + process.env[LUMIGO_SUPPORT_LARGE_INVOCATIONS] = 'true'; + expect(getMaxSizeForStoredSpansInMemory()).toBe(valueBefore + 100); }); test('getConnectionTimeout => simple flow', () => { diff --git a/src/utils.ts b/src/utils.ts index be592c1a9..78d54bfb5 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -343,8 +343,6 @@ export const isLambdaWrapped = (): boolean => validateEnvVar(WRAPPED_FLAG); export const shouldPropagateW3C = (): boolean => !validateEnvVar(LUMIGO_PROPAGATE_W3C, 'FALSE'); -export const shouldTryZip = (): boolean => validateEnvVar(LUMIGO_SUPPORT_LARGE_INVOCATIONS); - export const setLambdaWrapped = (): void => { process.env[WRAPPED_FLAG] = 'TRUE'; }; @@ -379,20 +377,6 @@ export const getMaxRequestSize = () => TracerGlobals.getTracerInputs().maxSizeFo export const getMaxRequestSizeOnError = () => TracerGlobals.getTracerInputs().maxSizeForRequestOnError; -/** - * The maximum size of all spans stored in memory before sending them to lumigo. - * This limit is in place to prevent storing too many spans in memory and causing OOM errors. - * Note: when the invocation ends and the spans are processed before sending to Lumigo, more processing and truncating - * might take place - * @returns {number} The maximum size of all spans stored in memory in bytes - */ -export const getStoredSpansMaxSize = (): number => { - return ( - parseInt(process.env[LUMIGO_STORED_SPANS_MAX_SIZE_BYTES_ENV_VAR]) || - getMaxRequestSizeOnError() * 10 - ); -}; - export const getInvokedArn = () => { // @ts-ignore return TracerGlobals.getHandlerInputs().context.invokedFunctionArn || ''; @@ -689,6 +673,31 @@ export const filterObjectKeys = ( return Object.assign(cur, { [key]: obj[key] }); }, {}); +export const shouldTryZip = () => validateEnvVar(LUMIGO_SUPPORT_LARGE_INVOCATIONS); + +const getLumigoStoredSpansMaxSizeBytesOverrideValue = () => { + const value = process.env[LUMIGO_STORED_SPANS_MAX_SIZE_BYTES_ENV_VAR]; + if (value && shouldTryZip()) { + const parsedValue = parseInt(value); + if (!isNaN(parsedValue)) { + return parsedValue; + } + } + return undefined; +}; + +/** + * The maximum size of all spans stored in memory before sending them to lumigo. + * This limit is in place to prevent storing too many spans in memory and causing OOM errors. + * Note: when the invocation ends and the spans are processed before sending to Lumigo, more processing and truncating + * might take place + * @returns number maximum size in bytes + */ +export const getMaxSizeForStoredSpansInMemory = (): number => { + const overrideValue = getLumigoStoredSpansMaxSizeBytesOverrideValue(); + return overrideValue || TracerGlobals.getTracerInputs().maxSizeForStoredSpansInMemory; +}; + export const isLambdaTraced = () => isAwsEnvironment() && !isSwitchedOff(); export const getRequestBodyMaskingRegex = (): string | undefined => process.env[LUMIGO_SECRET_MASKING_REGEX_HTTP_REQUEST_BODIES]; From cc7e7b2a60c0ac40f3ecac6485f667769bd8d1e0 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu <140942608+sagivoululumigo@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:39:20 +0300 Subject: [PATCH 11/14] Update src/globals.ts Co-authored-by: Harel Moshe --- src/globals.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/globals.ts b/src/globals.ts index bbf340f00..59268bb5b 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -26,8 +26,7 @@ export const MIN_TRACER_ADDED_DURATION_ALLOWED = 200; const warnSpansSizeOnce = runOneTimeWrapper((threshold: number, currentSize: number) => { logger.info( - `Lumigo tracer is no longer collecting data on the invocation, because it reached the maximum ` + - `size of calls collected of ${threshold} bytes (current size is ${currentSize} bytes)` + `Lumigo tracer is no longer collecting data on the invocation - maximum size of total spans collected (${threshold} bytes allowed, current size is ${currentSize} bytes)` ); }, {}); From 006e9729fbad61f55a86ba22a77fc74874bd5ba4 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu <140942608+sagivoululumigo@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:39:57 +0300 Subject: [PATCH 12/14] Update src/globals.ts Co-authored-by: Harel Moshe --- src/globals.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/globals.ts b/src/globals.ts index 59268bb5b..b97f63702 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -42,7 +42,7 @@ export class SpansContainer { } // Memory optimization, take up to 10x maxSize because of smart span selection logic const maxSpansSize = getMaxSizeForStoredSpansInMemory(); - if (spanHasErrors(span) || maxSpansSize > this.currentSpansSize) { + if (spanHasErrors(span) || this.currentSpansSize <= maxSpansSize) { this.spans[span.id] = span; this.currentSpansSize += getJSONBase64Size(span); logger.debug('Span created', span); From e3b9f1dc3b01879b87254718f575f4ec2e45fd74 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Thu, 11 Jul 2024 13:09:55 +0300 Subject: [PATCH 13/14] fix: add total spans limit to debug log msg --- src/globals.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/globals.ts b/src/globals.ts index b97f63702..d662e835b 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -51,6 +51,7 @@ export class SpansContainer { logger.debug('Span was not added due to size limitations', { currentSpansSize: this.currentSpansSize, + maxSpansSize, }); warnSpansSizeOnce(maxSpansSize, this.currentSpansSize); return false; From 18b19f3897a46dc1cca3fe5136b8ba9a0c918b96 Mon Sep 17 00:00:00 2001 From: Sagiv Oulu Date: Thu, 11 Jul 2024 13:14:45 +0300 Subject: [PATCH 14/14] fix: var name used the word size twice --- src/globals.test.js | 6 +++--- src/globals.ts | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/globals.test.js b/src/globals.test.js index 97da0584a..552fa76ee 100644 --- a/src/globals.test.js +++ b/src/globals.test.js @@ -3,7 +3,7 @@ import * as globals from './globals'; import { DEFAULT_MAX_SIZE_FOR_REQUEST, DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR, - DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, + DEFAULT_MAX_SIZE_FOR_SPANS_STORED_IN_MEMORY, MAX_TRACER_ADDED_DURATION_ALLOWED, } from './globals'; import { getMaxRequestSize } from './utils'; @@ -271,7 +271,7 @@ describe('globals', () => { isStepFunction: false, maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST, maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR, - maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, + maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_STORED_IN_MEMORY, }); }); @@ -315,7 +315,7 @@ describe('globals', () => { lambdaTimeout: MAX_TRACER_ADDED_DURATION_ALLOWED, maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST, maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR, - maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, + maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_STORED_IN_MEMORY, }); }); diff --git a/src/globals.ts b/src/globals.ts index d662e835b..9371353a2 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -19,7 +19,7 @@ const MAX_TAG_VALUE_LEN = 70; const ADD_TAG_ERROR_MSG_PREFIX = 'Skipping addExecutionTag: Unable to add tag'; export const DEFAULT_MAX_SIZE_FOR_REQUEST = 1024 * 500; export const DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR = 1024 * 990; -export const DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY = +export const DEFAULT_MAX_SIZE_FOR_SPANS_STORED_IN_MEMORY = DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR * 10; export const MAX_TRACER_ADDED_DURATION_ALLOWED = 750; export const MIN_TRACER_ADDED_DURATION_ALLOWED = 200; @@ -228,7 +228,7 @@ export const TracerGlobals = (() => { isStepFunction: false, maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST, maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR, - maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, + maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_STORED_IN_MEMORY, lambdaTimeout: MAX_TRACER_ADDED_DURATION_ALLOWED, }; @@ -257,7 +257,7 @@ export const TracerGlobals = (() => { maxSizeForRequest = null, maxSizeForRequestOnError = null, lambdaTimeout = MAX_TRACER_ADDED_DURATION_ALLOWED, - maxSizeForStoredSpansInMemory = DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, + maxSizeForStoredSpansInMemory = DEFAULT_MAX_SIZE_FOR_SPANS_STORED_IN_MEMORY, }: TracerOptions) => { const parsedMaxSizeForRequestOnError = maxSizeForRequestOnError || @@ -299,7 +299,7 @@ export const TracerGlobals = (() => { isStepFunction: false, maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST, maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR, - maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_SIZE_STORED_IN_MEMORY, + maxSizeForStoredSpansInMemory: DEFAULT_MAX_SIZE_FOR_SPANS_STORED_IN_MEMORY, }); return {