Skip to content

Commit

Permalink
feat: support bigger request size on errors & bigger entry size on error
Browse files Browse the repository at this point in the history
  • Loading branch information
@nadav3396
nadav3396 committed Jan 16, 2024
1 parent 0cad80b commit e6aa0f4
Show file tree
Hide file tree
Showing 7 changed files with 182 additions and 45 deletions.
11 changes: 10 additions & 1 deletion src/globals.test.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
import { ConsoleWritesForTesting } from '../testUtils/consoleMocker';
import * as globals from './globals';
import { DEFAULT_MAX_SIZE_FOR_REQUEST, MAX_TRACER_ADDED_DURATION_ALLOWED } from './globals';
import {
DEFAULT_MAX_SIZE_FOR_REQUEST,
DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR,
MAX_TRACER_ADDED_DURATION_ALLOWED,
} from './globals';
import { getMaxRequestSize } from './utils';

describe('globals', () => {
Expand Down Expand Up @@ -228,13 +232,15 @@ describe('globals', () => {
const edgeHost = 'zarathustra.com';
const isStepFunction = false;
const maxSizeForRequest = 1234;
const maxSizeForRequestOnError = 12345;
globals.TracerGlobals.setTracerInputs({
token,
debug,
edgeHost,
switchOff,
isStepFunction,
maxSizeForRequest,
maxSizeForRequestOnError,
});
expect(globals.TracerGlobals.getTracerInputs()).toEqual({
token,
Expand All @@ -244,6 +250,7 @@ describe('globals', () => {
isStepFunction,
lambdaTimeout: MAX_TRACER_ADDED_DURATION_ALLOWED,
maxSizeForRequest,
maxSizeForRequestOnError,
});
globals.TracerGlobals.clearTracerInputs();
expect(globals.TracerGlobals.getTracerInputs()).toEqual({
Expand All @@ -254,6 +261,7 @@ describe('globals', () => {
switchOff: false,
isStepFunction: false,
maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST,
maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR,
});
});

Expand Down Expand Up @@ -296,6 +304,7 @@ describe('globals', () => {
isStepFunction: false,
lambdaTimeout: MAX_TRACER_ADDED_DURATION_ALLOWED,
maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST,
maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR,
});
});

Expand Down
9 changes: 9 additions & 0 deletions src/globals.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const MAX_TAG_KEY_LEN = 50;
const MAX_TAG_VALUE_LEN = 70;
const ADD_TAG_ERROR_MSG_PREFIX = 'Skipping addExecutionTag: Unable to add tag';
export const DEFAULT_MAX_SIZE_FOR_REQUEST = 1024 * 500;
export const DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR = 1024 * 990;
export const MAX_TRACER_ADDED_DURATION_ALLOWED = 750;
export const MIN_TRACER_ADDED_DURATION_ALLOWED = 200;

Expand Down Expand Up @@ -199,6 +200,7 @@ export const TracerGlobals = (() => {
switchOff: false,
isStepFunction: false,
maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST,
maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR,
lambdaTimeout: MAX_TRACER_ADDED_DURATION_ALLOWED,
};

Expand All @@ -225,6 +227,7 @@ export const TracerGlobals = (() => {
switchOff = false,
stepFunction = false,
maxSizeForRequest = null,
maxSizeForRequestOnError = null,
lambdaTimeout = MAX_TRACER_ADDED_DURATION_ALLOWED,
}: TracerOptions) =>
Object.assign(tracerInputs, {
Expand All @@ -244,6 +247,11 @@ export const TracerGlobals = (() => {
(process.env['LUMIGO_MAX_SIZE_FOR_REQUEST']
? parseInt(process.env.LUMIGO_MAX_SIZE_FOR_REQUEST)
: DEFAULT_MAX_SIZE_FOR_REQUEST),
maxSizeForRequestOnError:
maxSizeForRequestOnError ||
(process.env['LUMIGO_MAX_SIZE_FOR_REQUEST_ON_ERROR']
? parseInt(process.env.LUMIGO_MAX_SIZE_FOR_REQUEST_ON_ERROR)
: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR),
});

const getTracerInputs = () => tracerInputs;
Expand All @@ -256,6 +264,7 @@ export const TracerGlobals = (() => {
switchOff: false,
isStepFunction: false,
maxSizeForRequest: DEFAULT_MAX_SIZE_FOR_REQUEST,
maxSizeForRequestOnError: DEFAULT_MAX_SIZE_FOR_REQUEST_ON_ERROR,
});

return {
Expand Down
147 changes: 123 additions & 24 deletions src/reporter.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,79 @@ describe('reporter', () => {
expect(sentSpans).toEqual([spans]);
});

test(`forgeAndScrubRequestBody - use requestSizeOnError when having error span`, async () => {
const end = {
end: 'dummyEnd',
type: FUNCTION_SPAN,
envs: { firstEnvKey: 'First environment variable value' },
};
const error = {
dummy: 'dummy',
type: HTTP_SPAN,
error: 'error',
info: {
httpInfo: {
host: 'your.mind.com',
request: {
host: 'your.mind.com',
headers: {
'content-type': 'json',
},
body: JSON.stringify({
body: 'no response because we have an error',
}),
},
},
},
};

const spans = [error, end];
const expectedSpans = [error, end];
const size = getJSONBase64Size(expectedSpans);
TracerGlobals.setTracerInputs({ maxSizeForRequest: size - 30, maxSizeForRequestOnError: size });

await reporter.sendSpans(spans);

const sentSpans = AxiosMocker.getSentSpans();
expect(sentSpans).toEqual([expectedSpans]);
});

test(`forgeAndScrubRequestBody - don't use requestSizeOnError when all spans succeed`, async () => {
const end = {
end: 'dummyEnd',
type: FUNCTION_SPAN,
envs: { firstEnvKey: 'First environment variable value' },
};
const dummy = {
dummy: 'dummy',
type: HTTP_SPAN,
info: {
httpInfo: {
host: 'your.mind.com',
request: {
host: 'your.mind.com',
headers: {
'content-type': 'json',
},
body: JSON.stringify({
body: 'no response because we have an error',
}),
},
},
},
};

const spans = [dummy, dummy, end];
const expectedSpans = [end, dummy];
const size = getJSONBase64Size(expectedSpans);
TracerGlobals.setTracerInputs({ maxSizeForRequest: size, maxSizeForRequestOnError: size * 2 });

await reporter.sendSpans(spans);

const sentSpans = AxiosMocker.getSentSpans();
expect(sentSpans).toEqual([expectedSpans]);
});

test(`forgeAndScrubRequestBody - with smart prioritization only metadata`, async () => {
const dummy = {
dummy: 'dummy',
Expand Down Expand Up @@ -177,7 +250,7 @@ describe('reporter', () => {
const spans = [dummy, error, end];
const expectedSpans = [endMetadata, errorMetadata, dummyMetadata];
const size = getJSONBase64Size(expectedSpans);
TracerGlobals.setTracerInputs({ maxSizeForRequest: size });
TracerGlobals.setTracerInputs({ maxSizeForRequest: size, maxSizeForRequestOnError: size });

await reporter.sendSpans(spans);

Expand Down Expand Up @@ -241,7 +314,7 @@ describe('reporter', () => {
const spans = [dummy, error, end];
const expectedSpans = [end, error, dummyMetadata];
const size = getJSONBase64Size(expectedSpans);
TracerGlobals.setTracerInputs({ maxSizeForRequest: size });
TracerGlobals.setTracerInputs({ maxSizeForRequest: size, maxSizeForRequestOnError: size });

await reporter.sendSpans(spans);

Expand All @@ -261,9 +334,9 @@ describe('reporter', () => {
const expectedResult = [dummyEnd, dummy];
const expectedResultSize = getJSONBase64Size(expectedResult);

expect(reporter.forgeAndScrubRequestBody(spans, expectedResultSize)).toEqual(
JSON.stringify(expectedResult)
);
expect(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
).toEqual(JSON.stringify(expectedResult));
});

describe('forgeAndScrubRequestBody parsing tests', () => {
Expand Down Expand Up @@ -321,7 +394,11 @@ describe('reporter', () => {
];
const expectedResultSize = getJSONBase64Size(spans);

const actual = reporter.forgeAndScrubRequestBody(spans, expectedResultSize);
const actual = reporter.forgeAndScrubRequestBody(
spans,
expectedResultSize,
expectedResultSize
);
expect(actual).toEqual(JSON.stringify(expectedResult));
});

Expand Down Expand Up @@ -365,7 +442,9 @@ describe('reporter', () => {
];
const expectedResultSize = getJSONBase64Size(spans);
process.env.LUMIGO_DOMAINS_SCRUBBER = '["mind"]';
const actual = JSON.parse(reporter.forgeAndScrubRequestBody(spans, expectedResultSize));
const actual = JSON.parse(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
);
expect(actual).toEqual(expected);
});
test('forgeAndScrubRequestBody', () => {
Expand Down Expand Up @@ -410,7 +489,9 @@ describe('reporter', () => {
];
const expectedResultSize = getJSONBase64Size(spans);

const actual = JSON.parse(reporter.forgeAndScrubRequestBody(spans, expectedResultSize));
const actual = JSON.parse(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
);
expect(actual).toEqual(expected);
});

Expand Down Expand Up @@ -465,7 +546,9 @@ describe('reporter', () => {
];
const expectedResultSize = getJSONBase64Size(spans);

const actual = JSON.parse(reporter.forgeAndScrubRequestBody(spans, expectedResultSize));
const actual = JSON.parse(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
);
expect(actual).toEqual(expected);
});

Expand Down Expand Up @@ -522,7 +605,9 @@ describe('reporter', () => {
];
const expectedResultSize = getJSONBase64Size(spans);

const actual = JSON.parse(reporter.forgeAndScrubRequestBody(spans, expectedResultSize));
const actual = JSON.parse(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
);
expect(actual).toEqual(expected);
});

Expand Down Expand Up @@ -577,7 +662,9 @@ describe('reporter', () => {
];
const expectedResultSize = getJSONBase64Size(spans);

const actual = JSON.parse(reporter.forgeAndScrubRequestBody(spans, expectedResultSize));
const actual = JSON.parse(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
);
expect(actual).toEqual(expected);
});

Expand Down Expand Up @@ -634,7 +721,9 @@ describe('reporter', () => {
];
const expectedResultSize = getJSONBase64Size(spans);

const actual = JSON.parse(reporter.forgeAndScrubRequestBody(spans, expectedResultSize));
const actual = JSON.parse(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
);
expect(actual).toEqual(expected);
});
test('forgeAndScrubRequestBody short response', () => {
Expand Down Expand Up @@ -679,7 +768,9 @@ describe('reporter', () => {
];
const expectedResultSize = getJSONBase64Size(spans);

const actual = JSON.parse(reporter.forgeAndScrubRequestBody(spans, expectedResultSize));
const actual = JSON.parse(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
);
expect(actual).toEqual(expected);
});

Expand Down Expand Up @@ -725,7 +816,9 @@ describe('reporter', () => {
];
const expectedResultSize = getJSONBase64Size(spans);

const actual = JSON.parse(reporter.forgeAndScrubRequestBody(spans, expectedResultSize));
const actual = JSON.parse(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
);
expect(actual).toEqual(expected);
});

Expand Down Expand Up @@ -775,7 +868,9 @@ describe('reporter', () => {
];
const expectedResultSize = getJSONBase64Size(spans);

const actual = JSON.parse(reporter.forgeAndScrubRequestBody(spans, expectedResultSize));
const actual = JSON.parse(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
);
expect(actual).toEqual(expected);
});

Expand Down Expand Up @@ -835,10 +930,14 @@ describe('reporter', () => {
const expectedResultSizeFail = getJSONBase64Size(spansFail);

const spanSuccess = JSON.parse(
reporter.forgeAndScrubRequestBody(spansSuccess, expectedResultSizeSuccess)
reporter.forgeAndScrubRequestBody(
spansSuccess,
expectedResultSizeSuccess,
expectedResultSizeSuccess
)
)[0];
const spanError = JSON.parse(
reporter.forgeAndScrubRequestBody(spansFail, expectedResultSizeFail)
reporter.forgeAndScrubRequestBody(spansFail, expectedResultSizeFail, expectedResultSizeFail)
)[0];
expect(spanError.info.httpInfo.request.body.length).toBeGreaterThan(
spanSuccess.info.httpInfo.request.body.length * 1.8 + 1
Expand All @@ -864,9 +963,9 @@ describe('reporter', () => {
const expectedResult = [JSON.parse(JSON.stringify(end)), JSON.parse(JSON.stringify(error))];
const expectedResultSize = getJSONBase64Size(expectedResult);

expect(reporter.forgeAndScrubRequestBody(spans, expectedResultSize)).toEqual(
JSON.stringify(expectedResult)
);
expect(
reporter.forgeAndScrubRequestBody(spans, expectedResultSize, expectedResultSize)
).toEqual(JSON.stringify(expectedResult));
});

test('forgeRequestBody - cut spans - skip initiate stringify (performance boost)', async () => {
Expand All @@ -886,12 +985,12 @@ describe('reporter', () => {
const error = 'error';
const spans = [{ dummy }, { dummy, error }, { dummyEnd }];

expect(reporter.forgeAndScrubRequestBody(spans, 100)).toEqual(JSON.stringify(spans));
expect(reporter.forgeAndScrubRequestBody([], 100)).toEqual(undefined);
expect(reporter.forgeAndScrubRequestBody(spans, 100, 100)).toEqual(JSON.stringify(spans));
expect(reporter.forgeAndScrubRequestBody([], 100, 100)).toEqual(undefined);
});

test('forgeRequestBody - empty list', async () => {
expect(reporter.forgeAndScrubRequestBody([], 100)).toEqual(undefined);
expect(reporter.forgeAndScrubRequestBody([], 100, 100)).toEqual(undefined);
});

test('scrubSpans missing http fields', () => {
Expand Down Expand Up @@ -1039,7 +1138,7 @@ describe('reporter', () => {
expect(logs.length).toEqual(2);
expect(logs[0].msg).toEqual('#LUMIGO# - WARNING - "Error in Lumigo tracer"');
expect(logs[1].msg).toEqual('#LUMIGO# - WARNING - "Error in Lumigo tracer"');
expect(JSON.parse(logs[0].obj).message).toEqual('resultSpans.filter is not a function');
expect(JSON.parse(logs[0].obj).message).toEqual('spans.some is not a function');
expect(JSON.parse(logs[1].obj).message).toEqual('spans.map is not a function');
expect(JSON.parse(logs[0].obj).stack).toBeTruthy();
expect(JSON.parse(logs[1].obj).stack).toBeTruthy();
Expand Down
Loading

0 comments on commit e6aa0f4

Please sign in to comment.