Skip to content

Commit

Permalink
feat(lib-dynamodb): add support for imprecise numbers and custom numb…
Browse files Browse the repository at this point in the history
…er retrieval (#6644)
  • Loading branch information
RanVaknin authored Nov 11, 2024
1 parent 17b37b7 commit 4e2f525
Show file tree
Hide file tree
Showing 7 changed files with 130 additions and 14 deletions.
66 changes: 63 additions & 3 deletions lib/lib-dynamodb/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,15 +120,26 @@ export interface marshallOptions {
* but false if directly using the marshall function (backwards compatibility).
*/
convertTopLevelContainer?: boolean;
/**
* Whether to allow numbers beyond Number.MAX_SAFE_INTEGER during marshalling.
* When set to true, allows numbers that may lose precision when converted to JavaScript numbers.
* When false (default), throws an error if a number exceeds Number.MAX_SAFE_INTEGER to prevent
* unintended loss of precision. Consider using the NumberValue type from @aws-sdk/lib-dynamodb
* for precise handling of large numbers.
*/
allowImpreciseNumbers?: boolean;
}

export interface unmarshallOptions {
/**
* Whether to return numbers as a string instead of converting them to native JavaScript numbers.
* Whether to modify how numbers are unmarshalled from DynamoDB.
* When set to true, returns numbers as NumberValue instances instead of native JavaScript numbers.
* This allows for the safe round-trip transport of numbers of arbitrary size.
*
* If a function is provided, it will be called with the string representation of numbers to handle
* custom conversions (e.g., using BigInt or decimal libraries).
*/
wrapNumbers?: boolean;

wrapNumbers?: boolean | ((value: string) => number | bigint | NumberValue | any);
/**
* When true, skip wrapping the data in `{ M: data }` before converting.
*
Expand Down Expand Up @@ -235,10 +246,59 @@ const response = await client.get({
const value = response.Item.bigNumber;
```

You can also provide a custom function to handle number conversion during unmarshalling:

```typescript
const client = DynamoDBDocument.from(new DynamoDB({}), {
unmarshallOptions: {
// Use BigInt for all numbers
wrapNumbers: (str) => BigInt(str),
},
});

const response = await client.get({
Key: { id: 1 },
});

// Numbers in response will be BigInt instead of NumberValue or regular numbers
```

`NumberValue` does not provide a way to do mathematical operations on itself.
To do mathematical operations, take the string value of `NumberValue` by calling
`.toString()` and supply it to your chosen big number implementation.

The client protects against precision loss by throwing an error on large numbers, but you can either
allow imprecise values with `allowImpreciseNumbers` or maintain exact precision using `NumberValue`.

```typescript
const preciseValue = "34567890123456789012345678901234567890";

// 1. Default behavior - will throw error
await client.send(
new PutCommand({
TableName: "Table",
Item: {
id: "1",
number: Number(preciseValue), // Throws error: Number is greater than Number.MAX_SAFE_INTEGER
},
})
);

// 2. Using allowImpreciseNumbers - will store but loses precision (mimics the v2 implicit behavior)
const impreciseClient = DynamoDBDocumentClient.from(new DynamoDBClient({}), {
marshallOptions: { allowImpreciseNumbers: true },
});
await impreciseClient.send(
new PutCommand({
TableName: "Table",
Item: {
id: "2",
number: Number(preciseValue), // Loses precision 34567890123456790000000000000000000000n
},
})
);
```

### Client and Command middleware stacks

As with other AWS SDK for JavaScript v3 clients, you can apply middleware functions
Expand Down
28 changes: 28 additions & 0 deletions packages/util-dynamodb/src/convertToAttr.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -595,4 +595,32 @@ describe("convertToAttr", () => {
expect(convertToAttr(new Date(), { convertClassInstanceToMap: true })).toEqual({ M: {} });
});
});

describe("imprecise numbers", () => {
const impreciseNumbers = [
{ val: 1.23e40, str: "1.23e+40" }, // https://github.com/aws/aws-sdk-js-v3/issues/6571
{ val: Number.MAX_VALUE, str: Number.MAX_VALUE.toString() },
{ val: Number.MAX_SAFE_INTEGER + 1, str: (Number.MAX_SAFE_INTEGER + 1).toString() },
];

describe("without allowImpreciseNumbers", () => {
impreciseNumbers.forEach(({ val }) => {
it(`throws for imprecise number: ${val}`, () => {
expect(() => {
convertToAttr(val);
}).toThrowError(
`Number ${val.toString()} is greater than Number.MAX_SAFE_INTEGER. Use NumberValue from @aws-sdk/lib-dynamodb.`
);
});
});
});

describe("with allowImpreciseNumbers", () => {
impreciseNumbers.forEach(({ val, str }) => {
it(`allows imprecise number: ${val}`, () => {
expect(convertToAttr(val, { allowImpreciseNumbers: true })).toEqual({ N: str });
});
});
});
});
});
17 changes: 10 additions & 7 deletions packages/util-dynamodb/src/convertToAttr.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export const convertToAttr = (data: NativeAttributeValue, options?: marshallOpti
} else if (typeof data === "boolean" || data?.constructor?.name === "Boolean") {
return { BOOL: data.valueOf() };
} else if (typeof data === "number" || data?.constructor?.name === "Number") {
return convertToNumberAttr(data);
return convertToNumberAttr(data, options);
} else if (data instanceof NumberValue) {
return data.toAttributeValue();
} else if (typeof data === "bigint") {
Expand Down Expand Up @@ -91,7 +91,7 @@ const convertToSetAttr = (
} else if (typeof item === "number") {
return {
NS: Array.from(setToOperate)
.map(convertToNumberAttr)
.map((num) => convertToNumberAttr(num, options))
.map((item) => item.N),
};
} else if (typeof item === "bigint") {
Expand Down Expand Up @@ -160,17 +160,20 @@ const validateBigIntAndThrow = (errorPrefix: string) => {
throw new Error(`${errorPrefix} Use NumberValue from @aws-sdk/lib-dynamodb.`);
};

const convertToNumberAttr = (num: number | Number): { N: string } => {
const convertToNumberAttr = (num: number | Number, options?: marshallOptions): { N: string } => {
if (
[Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]
.map((val) => val.toString())
.includes(num.toString())
) {
throw new Error(`Special numeric value ${num.toString()} is not allowed`);
} else if (num > Number.MAX_SAFE_INTEGER) {
validateBigIntAndThrow(`Number ${num.toString()} is greater than Number.MAX_SAFE_INTEGER.`);
} else if (num < Number.MIN_SAFE_INTEGER) {
validateBigIntAndThrow(`Number ${num.toString()} is lesser than Number.MIN_SAFE_INTEGER.`);
} else if (!options?.allowImpreciseNumbers) {
// Only perform these checks if allowImpreciseNumbers is false
if (num > Number.MAX_SAFE_INTEGER) {
validateBigIntAndThrow(`Number ${num.toString()} is greater than Number.MAX_SAFE_INTEGER.`);
} else if (num < Number.MIN_SAFE_INTEGER) {
validateBigIntAndThrow(`Number ${num.toString()} is lesser than Number.MIN_SAFE_INTEGER.`);
}
}
return { N: num.toString() };
};
Expand Down
11 changes: 11 additions & 0 deletions packages/util-dynamodb/src/convertToNative.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,17 @@ describe("convertToNative", () => {
}).toThrowError(`${numString} can't be converted to BigInt. Set options.wrapNumbers to get string value.`);
});
});

it("handles custom wrapNumbers function", () => {
expect(
convertToNative(
{ N: "124" },
{
wrapNumbers: (str: string) => Number(str) / 2,
}
)
).toEqual(62);
});
});

describe("binary", () => {
Expand Down
4 changes: 3 additions & 1 deletion packages/util-dynamodb/src/convertToNative.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,12 @@ export const convertToNative = (data: AttributeValue, options?: unmarshallOption
};

const convertNumber = (numString: string, options?: unmarshallOptions): number | bigint | NumberValue => {
if (typeof options?.wrapNumbers === "function") {
return options?.wrapNumbers(numString);
}
if (options?.wrapNumbers) {
return NumberValue.from(numString);
}

const num = Number(numString);
const infinityValues = [Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY];
const isLargeFiniteNumber =
Expand Down
8 changes: 8 additions & 0 deletions packages/util-dynamodb/src/marshall.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,14 @@ export interface marshallOptions {
* but false if directly using the marshall function (backwards compatibility).
*/
convertTopLevelContainer?: boolean;
/**
* Whether to allow numbers beyond Number.MAX_SAFE_INTEGER during marshalling.
* When set to true, allows numbers that may lose precision when converted to JavaScript numbers.
* When false (default), throws an error if a number exceeds Number.MAX_SAFE_INTEGER to prevent
* unintended loss of precision. Consider using the NumberValue type from @aws-sdk/lib-dynamodb
* for precise handling of large numbers.
*/
allowImpreciseNumbers?: boolean;
}

/**
Expand Down
10 changes: 7 additions & 3 deletions packages/util-dynamodb/src/unmarshall.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,21 @@ import { AttributeValue } from "@aws-sdk/client-dynamodb";

import { convertToNative } from "./convertToNative";
import { NativeAttributeValue } from "./models";
import { NumberValue } from "./NumberValue";

/**
* An optional configuration object for `convertToNative`
*/
export interface unmarshallOptions {
/**
* Whether to return numbers as a string instead of converting them to native JavaScript numbers.
* Whether to modify how numbers are unmarshalled from DynamoDB.
* When set to true, returns numbers as NumberValue instances instead of native JavaScript numbers.
* This allows for the safe round-trip transport of numbers of arbitrary size.
*
* If a function is provided, it will be called with the string representation of numbers to handle
* custom conversions (e.g., using BigInt or decimal libraries).
*/
wrapNumbers?: boolean;

wrapNumbers?: boolean | ((value: string) => number | bigint | NumberValue | any);
/**
* When true, skip wrapping the data in `{ M: data }` before converting.
*
Expand Down

0 comments on commit 4e2f525

Please sign in to comment.