From 076339e095ad015077793580357fcc9dafc7bb9b Mon Sep 17 00:00:00 2001 From: extJo Date: Wed, 8 Jul 2020 12:32:39 +0900 Subject: [PATCH 1/7] Add scanAll at fullPrimaryKey --- src/query/__test__/full_primary_key_spec.ts | 75 +++++++++++++++++++++ src/query/full_primary_key.ts | 42 ++++++++++++ 2 files changed, 117 insertions(+) diff --git a/src/query/__test__/full_primary_key_spec.ts b/src/query/__test__/full_primary_key_spec.ts index 988e06a..0854fea 100644 --- a/src/query/__test__/full_primary_key_spec.ts +++ b/src/query/__test__/full_primary_key_spec.ts @@ -209,6 +209,81 @@ describe("FullPrimaryKey", () => { expect(res.records[1].title).to.eq("abc"); }); }); + + describe("#scanAll", () => { + it("should find all items", async () => { + await Card.metadata.connection.documentClient.put({ + TableName: Card.metadata.name, + Item: { + id: 10, + title: "abc", + }, + }).promise(); + await Card.metadata.connection.documentClient.put({ + TableName: Card.metadata.name, + Item: { + id: 10, + title: "abd", + }, + }).promise(); + await Card.metadata.connection.documentClient.put({ + TableName: Card.metadata.name, + Item: { + id: 10, + title: "aba", + }, + }).promise(); + + const res = await primaryKey.scanAll({}); + + expect(res.records.length).to.eq(3); + // Ordered by range key since it's "scan" + expect(res.records[0].title).to.eq("aba"); + expect(res.records[1].title).to.eq("abc"); + expect(res.records[2].title).to.eq("abd"); + }); + + it("should find all items with parallelize", async () => { + await Card.metadata.connection.documentClient.put({ + TableName: Card.metadata.name, + Item: { + id: 10, + title: "abc", + }, + }).promise(); + await Card.metadata.connection.documentClient.put({ + TableName: Card.metadata.name, + Item: { + id: 10, + title: "abd", + }, + }).promise(); + await Card.metadata.connection.documentClient.put({ + TableName: Card.metadata.name, + Item: { + id: 10, + title: "aba", + }, + }).promise(); + await Card.metadata.connection.documentClient.put({ + TableName: Card.metadata.name, + Item: { + id: 10, + title: "ccc", + }, + }).promise(); + + const res = await primaryKey.scanAll({ parallelize: 3 }); + + expect(res.records.length).to.eq(4); + // Ordered by range key since it's "scan" + expect(res.records[0].title).to.eq("aba"); + expect(res.records[1].title).to.eq("abc"); + expect(res.records[2].title).to.eq("abd"); + expect(res.records[3].title).to.eq("ccc"); + }); + }); + describe("#update", () => { it("should be able to update items", async () => { await primaryKey.update(10, "abc", { count: ["ADD", 1] }); diff --git a/src/query/full_primary_key.ts b/src/query/full_primary_key.ts index 364b892..db38a4c 100644 --- a/src/query/full_primary_key.ts +++ b/src/query/full_primary_key.ts @@ -186,6 +186,48 @@ export class FullPrimaryKey { }; } + async scanAll(options: { + parallelize?: number; + limit?: number; + }) { + if (options.parallelize && options.parallelize < 1) { + throw new Error("Parallelize value at scanAll always positive number"); + } + const buffer: T[] = []; + const totalSegments = options.parallelize || 1; + const scanners = _.times(totalSegments) + .map((i) => (async (exclusiveStartKey?: DynamoDB.DocumentClient.Key) => + await this.scan({ + limit: options.limit, + totalSegments, + segment: i, + exclusiveStartKey, + })), + ); + let lastEvaluatedKeys = new Array(totalSegments); + _.fill(lastEvaluatedKeys, undefined); + + do { + const results = await Promise.all( + _.times(totalSegments) + .map((i) => scanners[i](lastEvaluatedKeys[i])), + ); + + buffer.push( + ..._.chain(results) + .map(({ records }) => records) + .flatten() + .value(), + ); + lastEvaluatedKeys = results.map(({ lastEvaluatedKey }) => lastEvaluatedKey); + } while(_.compact(lastEvaluatedKeys).length); + + return { + records: buffer, + count: buffer.length, + }; + } + async update( hashKey: HashKeyType, sortKey: RangeKeyType, From a00a6d9c6447cbefbf23b47fd40940aa10d4580e Mon Sep 17 00:00:00 2001 From: extJo Date: Wed, 8 Jul 2020 12:33:00 +0900 Subject: [PATCH 2/7] Add scanAll at hashPrimaryKey --- src/query/__test__/hash_primary_key_spec.ts | 51 +++++++++++++++++++++ src/query/hash_primary_key.ts | 43 +++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/src/query/__test__/hash_primary_key_spec.ts b/src/query/__test__/hash_primary_key_spec.ts index cc3ce4c..cd5fc4d 100644 --- a/src/query/__test__/hash_primary_key_spec.ts +++ b/src/query/__test__/hash_primary_key_spec.ts @@ -91,6 +91,57 @@ describe("HashPrimaryKey", () => { }); }); + describe("#scanAll", () => { + it("should find all items", async () => { + async function createCard() { + const card = new Card(); + card.id = faker.random.number(); + await card.save(); + return card; + } + + const cards = [ + await createCard(), + await createCard(), + await createCard(), + await createCard(), + ]; + + const res = await primaryKey.scanAll({}); + + const ids = _.sortBy( + res.records, + (item) => item.id, + ).map((c) => c.id); + + expect(ids).to.deep.eq( _.sortBy(cards.map((c) => c.id), (i) => i) ); + }); + + it("should find all items with parallelize", async () => { + async function createCard() { + const card = new Card(); + card.id = faker.random.number(); + await card.save(); + return card; + } + + const cards = [ + await createCard(), + await createCard(), + await createCard(), + await createCard(), + ]; + + const res = await primaryKey.scanAll({ parallelize: 3 }); + + const ids = _.sortBy( + res.records, + (item) => item.id, + ).map((c) => c.id); + + expect(ids).to.deep.eq( _.sortBy(cards.map((c) => c.id), (i) => i) ); + }); + }); describe("#batchGet", async () => { it ("should return results in order", async () => { diff --git a/src/query/hash_primary_key.ts b/src/query/hash_primary_key.ts index c5f1004..d8db027 100644 --- a/src/query/hash_primary_key.ts +++ b/src/query/hash_primary_key.ts @@ -1,4 +1,5 @@ import { DynamoDB } from 'aws-sdk'; +import * as _ from 'lodash'; import * as Codec from '../codec'; import * as Metadata from '../metadata'; @@ -66,6 +67,48 @@ export class HashPrimaryKey { }; } + async scanAll(options: { + parallelize?: number; + limit?: number; + }) { + if (options.parallelize && options.parallelize < 1) { + throw new Error("Parallelize value at scanAll always positive number"); + } + const buffer: T[] = []; + const totalSegments = options.parallelize || 1; + const scanners = _.times(totalSegments) + .map((i) => (async (exclusiveStartKey?: DynamoDB.DocumentClient.Key) => + await this.scan({ + limit: options.limit, + totalSegments, + segment: i, + exclusiveStartKey, + })), + ); + let lastEvaluatedKeys = new Array(totalSegments); + _.fill(lastEvaluatedKeys, undefined); + + do { + const results = await Promise.all( + _.times(totalSegments) + .map((i) => scanners[i](lastEvaluatedKeys[i])), + ); + + buffer.push( + ..._.chain(results) + .map(({ records }) => records) + .flatten() + .value(), + ); + lastEvaluatedKeys = results.map(({ lastEvaluatedKey }) => lastEvaluatedKey); + } while(_.compact(lastEvaluatedKeys).length); + + return { + records: buffer, + count: buffer.length, + }; + } + async batchGet(keys: HashKeyType[]) { const res = await batchGetTrim( this.tableClass.metadata.connection.documentClient, From d5e40e049c54baab29f666ef5625a304016c4cd5 Mon Sep 17 00:00:00 2001 From: extJo Date: Thu, 9 Jul 2020 15:07:04 +0900 Subject: [PATCH 3/7] Seperate scanAll for common function --- src/query/full_primary_key.ts | 39 ++++++------------------ src/query/global_secondary_index.ts | 42 ++++++++++++++++++++++++++ src/query/hash_primary_key.ts | 41 ++++++------------------- src/query/local_secondary_index.ts | 22 ++++++++++++++ src/query/scan_all.ts | 47 +++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 61 deletions(-) create mode 100644 src/query/scan_all.ts diff --git a/src/query/full_primary_key.ts b/src/query/full_primary_key.ts index db38a4c..40083b9 100644 --- a/src/query/full_primary_key.ts +++ b/src/query/full_primary_key.ts @@ -9,6 +9,7 @@ import * as Query from './query'; import { batchGetFull, batchGetTrim } from "./batch_get"; import { batchWrite } from "./batch_write"; +import { scanAll } from "./scan_all"; const HASH_KEY_REF = "#hk"; const HASH_VALUE_REF = ":hkv"; @@ -188,43 +189,21 @@ export class FullPrimaryKey { async scanAll(options: { parallelize?: number; - limit?: number; + scanBatchSize?: number; }) { if (options.parallelize && options.parallelize < 1) { throw new Error("Parallelize value at scanAll always positive number"); } - const buffer: T[] = []; - const totalSegments = options.parallelize || 1; - const scanners = _.times(totalSegments) - .map((i) => (async (exclusiveStartKey?: DynamoDB.DocumentClient.Key) => - await this.scan({ - limit: options.limit, - totalSegments, - segment: i, - exclusiveStartKey, - })), - ); - let lastEvaluatedKeys = new Array(totalSegments); - _.fill(lastEvaluatedKeys, undefined); - - do { - const results = await Promise.all( - _.times(totalSegments) - .map((i) => scanners[i](lastEvaluatedKeys[i])), - ); - buffer.push( - ..._.chain(results) - .map(({ records }) => records) - .flatten() - .value(), - ); - lastEvaluatedKeys = results.map(({ lastEvaluatedKey }) => lastEvaluatedKey); - } while(_.compact(lastEvaluatedKeys).length); + const res = await scanAll( + this.tableClass.metadata.connection.documentClient, + this.tableClass.metadata.name, + options, + ); return { - records: buffer, - count: buffer.length, + records: res.map((item) => Codec.deserialize(this.tableClass, item)), + count: res.length, }; } diff --git a/src/query/global_secondary_index.ts b/src/query/global_secondary_index.ts index a5cb462..e1862d6 100644 --- a/src/query/global_secondary_index.ts +++ b/src/query/global_secondary_index.ts @@ -7,6 +7,8 @@ import * as Codec from '../codec'; import * as Metadata from '../metadata'; import * as Query from './query'; +import { scanAll } from './scan_all'; + const HASH_KEY_REF = "#hk"; const HASH_VALUE_REF = ":hkv"; @@ -67,6 +69,26 @@ export class FullGlobalSecondaryIndex Codec.deserialize(this.tableClass, item)), + count: res.length, + }; + } } export class HashGlobalSecondaryIndex { @@ -103,4 +125,24 @@ export class HashGlobalSecondaryIndex { consumedCapacity: result.ConsumedCapacity, }; } + + async scanAll(options: { + parallelize?: number; + scanBatchSize?: number; + }) { + if (options.parallelize && options.parallelize < 1) { + throw new Error("Parallelize value at scanAll always positive number"); + } + + const res = await scanAll( + this.tableClass.metadata.connection.documentClient, + this.tableClass.metadata.name, + options, + ); + + return { + records: res.map((item) => Codec.deserialize(this.tableClass, item)), + count: res.length, + }; + } } diff --git a/src/query/hash_primary_key.ts b/src/query/hash_primary_key.ts index d8db027..89c4aa8 100644 --- a/src/query/hash_primary_key.ts +++ b/src/query/hash_primary_key.ts @@ -7,6 +7,7 @@ import { ITable, Table } from '../table'; import { batchGetFull, batchGetTrim } from "./batch_get"; import { batchWrite } from "./batch_write"; +import { scanAll } from './scan_all'; export class HashPrimaryKey { constructor( @@ -69,43 +70,21 @@ export class HashPrimaryKey { async scanAll(options: { parallelize?: number; - limit?: number; + scanBatchSize?: number; }) { if (options.parallelize && options.parallelize < 1) { throw new Error("Parallelize value at scanAll always positive number"); } - const buffer: T[] = []; - const totalSegments = options.parallelize || 1; - const scanners = _.times(totalSegments) - .map((i) => (async (exclusiveStartKey?: DynamoDB.DocumentClient.Key) => - await this.scan({ - limit: options.limit, - totalSegments, - segment: i, - exclusiveStartKey, - })), - ); - let lastEvaluatedKeys = new Array(totalSegments); - _.fill(lastEvaluatedKeys, undefined); - - do { - const results = await Promise.all( - _.times(totalSegments) - .map((i) => scanners[i](lastEvaluatedKeys[i])), - ); - - buffer.push( - ..._.chain(results) - .map(({ records }) => records) - .flatten() - .value(), - ); - lastEvaluatedKeys = results.map(({ lastEvaluatedKey }) => lastEvaluatedKey); - } while(_.compact(lastEvaluatedKeys).length); + + const res = await scanAll( + this.tableClass.metadata.connection.documentClient, + this.tableClass.metadata.name, + options, + ); return { - records: buffer, - count: buffer.length, + records: res.map((item) => Codec.deserialize(this.tableClass, item)), + count: res.length, }; } diff --git a/src/query/local_secondary_index.ts b/src/query/local_secondary_index.ts index 95c8eec..c7d2df9 100644 --- a/src/query/local_secondary_index.ts +++ b/src/query/local_secondary_index.ts @@ -7,6 +7,8 @@ import * as Codec from '../codec'; import * as Metadata from '../metadata'; import * as Query from './query'; +import { scanAll } from './scan_all'; + const HASH_KEY_REF = "#hk"; const HASH_VALUE_REF = ":hkv"; @@ -67,4 +69,24 @@ export class LocalSecondaryIndex { consumedCapacity: result.ConsumedCapacity, }; } + + async scanAll(options: { + parallelize?: number; + scanBatchSize?: number; + }) { + if (options.parallelize && options.parallelize < 1) { + throw new Error("Parallelize value at scanAll always positive number"); + } + + const res = await scanAll( + this.tableClass.metadata.connection.documentClient, + this.tableClass.metadata.name, + options, + ); + + return { + records: res.map((item) => Codec.deserialize(this.tableClass, item)), + count: res.length, + }; + } } diff --git a/src/query/scan_all.ts b/src/query/scan_all.ts new file mode 100644 index 0000000..4a38dca --- /dev/null +++ b/src/query/scan_all.ts @@ -0,0 +1,47 @@ +import { DynamoDB } from "aws-sdk"; +import * as _ from "lodash"; + +export async function scanAll( + documentClient: DynamoDB.DocumentClient, + tableName: string, + options: { + parallelize?: number; + scanBatchSize?: number; + }, +) { + if (options.parallelize && options.parallelize < 1) { + throw new Error("Parallelize value at scanAll always positive number"); + } + const buffer: DynamoDB.AttributeMap[] = []; + const totalSegments = options.parallelize || 1; + const scanners = _.times(totalSegments) + .map((i) => (async (exclusiveStartKey?: DynamoDB.DocumentClient.Key) => + await documentClient.scan({ + TableName: tableName, + Limit: options.scanBatchSize, + ExclusiveStartKey: exclusiveStartKey, + ReturnConsumedCapacity: "TOTAL", + TotalSegments: totalSegments, + Segment: i, + }).promise()), + ); + let lastEvaluatedKeys = new Array(totalSegments); + _.fill(lastEvaluatedKeys, undefined); + + do { + const results = await Promise.all( + _.times(totalSegments) + .map((i) => scanners[i](lastEvaluatedKeys[i])), + ); + + buffer.push( + ..._.chain(results) + .map(({ Items }) => (Items || [])) + .flatten() + .value(), + ); + lastEvaluatedKeys = results.map(({ LastEvaluatedKey }) => LastEvaluatedKey); + } while(_.compact(lastEvaluatedKeys).length); + + return buffer; +} From 3ba2bb53c8cb32ff236aefd093f1efe8ac640f65 Mon Sep 17 00:00:00 2001 From: extJo Date: Thu, 9 Jul 2020 15:07:40 +0900 Subject: [PATCH 4/7] change test logic --- src/query/__test__/hash_primary_key_spec.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/query/__test__/hash_primary_key_spec.ts b/src/query/__test__/hash_primary_key_spec.ts index cd5fc4d..815343d 100644 --- a/src/query/__test__/hash_primary_key_spec.ts +++ b/src/query/__test__/hash_primary_key_spec.ts @@ -105,9 +105,16 @@ describe("HashPrimaryKey", () => { await createCard(), await createCard(), await createCard(), + await createCard(), + await createCard(), + await createCard(), + await createCard(), + await createCard(), + await createCard(), + await createCard(), ]; - const res = await primaryKey.scanAll({}); + const res = await primaryKey.scanAll({ scanBatchSize: 1 }); const ids = _.sortBy( res.records, From 8871e516060c664b9555420793c811f13ef3af76 Mon Sep 17 00:00:00 2001 From: extJo Date: Thu, 9 Jul 2020 16:23:07 +0900 Subject: [PATCH 5/7] change scan all logic --- src/query/scan_all.ts | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/query/scan_all.ts b/src/query/scan_all.ts index 4a38dca..14debba 100644 --- a/src/query/scan_all.ts +++ b/src/query/scan_all.ts @@ -14,34 +14,38 @@ export async function scanAll( } const buffer: DynamoDB.AttributeMap[] = []; const totalSegments = options.parallelize || 1; - const scanners = _.times(totalSegments) - .map((i) => (async (exclusiveStartKey?: DynamoDB.DocumentClient.Key) => - await documentClient.scan({ - TableName: tableName, - Limit: options.scanBatchSize, - ExclusiveStartKey: exclusiveStartKey, - ReturnConsumedCapacity: "TOTAL", - TotalSegments: totalSegments, - Segment: i, - }).promise()), - ); - let lastEvaluatedKeys = new Array(totalSegments); - _.fill(lastEvaluatedKeys, undefined); + let scanners = _.times(totalSegments) + .map((i) => ({ + scanner: async (exclusiveStartKey?: DynamoDB.DocumentClient.Key) => + await documentClient.scan({ + TableName: tableName, + Limit: options.scanBatchSize, + ExclusiveStartKey: exclusiveStartKey, + ReturnConsumedCapacity: "TOTAL", + TotalSegments: totalSegments, + Segment: i, + }).promise(), + key: undefined as DynamoDB.DocumentClient.Key | undefined, + })); do { const results = await Promise.all( - _.times(totalSegments) - .map((i) => scanners[i](lastEvaluatedKeys[i])), + _.chain(scanners) + .map(async ({ scanner, key }) => await scanner(key)) + .value(), ); - buffer.push( ..._.chain(results) .map(({ Items }) => (Items || [])) .flatten() .value(), ); - lastEvaluatedKeys = results.map(({ LastEvaluatedKey }) => LastEvaluatedKey); - } while(_.compact(lastEvaluatedKeys).length); + scanners = scanners + .map(({ scanner }, index) => ({ + scanner, key: results[index].LastEvaluatedKey, + })) + .filter(({ key }) => !!key ); + } while(scanners.length); return buffer; } From fff2ba8a5797a5b24b33a66203195166de43ebd7 Mon Sep 17 00:00:00 2001 From: extJo Date: Thu, 9 Jul 2020 16:23:37 +0900 Subject: [PATCH 6/7] Remove throw error logic --- src/query/full_primary_key.ts | 4 ---- src/query/global_secondary_index.ts | 4 ---- src/query/hash_primary_key.ts | 4 ---- src/query/local_secondary_index.ts | 4 ---- 4 files changed, 16 deletions(-) diff --git a/src/query/full_primary_key.ts b/src/query/full_primary_key.ts index 40083b9..cb3bca3 100644 --- a/src/query/full_primary_key.ts +++ b/src/query/full_primary_key.ts @@ -191,10 +191,6 @@ export class FullPrimaryKey { parallelize?: number; scanBatchSize?: number; }) { - if (options.parallelize && options.parallelize < 1) { - throw new Error("Parallelize value at scanAll always positive number"); - } - const res = await scanAll( this.tableClass.metadata.connection.documentClient, this.tableClass.metadata.name, diff --git a/src/query/global_secondary_index.ts b/src/query/global_secondary_index.ts index e1862d6..a6a6b13 100644 --- a/src/query/global_secondary_index.ts +++ b/src/query/global_secondary_index.ts @@ -74,10 +74,6 @@ export class FullGlobalSecondaryIndex { parallelize?: number; scanBatchSize?: number; }) { - if (options.parallelize && options.parallelize < 1) { - throw new Error("Parallelize value at scanAll always positive number"); - } - const res = await scanAll( this.tableClass.metadata.connection.documentClient, this.tableClass.metadata.name, diff --git a/src/query/local_secondary_index.ts b/src/query/local_secondary_index.ts index c7d2df9..0626607 100644 --- a/src/query/local_secondary_index.ts +++ b/src/query/local_secondary_index.ts @@ -74,10 +74,6 @@ export class LocalSecondaryIndex { parallelize?: number; scanBatchSize?: number; }) { - if (options.parallelize && options.parallelize < 1) { - throw new Error("Parallelize value at scanAll always positive number"); - } - const res = await scanAll( this.tableClass.metadata.connection.documentClient, this.tableClass.metadata.name, From 9045b360f8c578a02221875e4f54070c04a66313 Mon Sep 17 00:00:00 2001 From: extJo Date: Thu, 9 Jul 2020 16:23:47 +0900 Subject: [PATCH 7/7] Add test --- src/query/__test__/full_primary_key_spec.ts | 4 - .../__test__/global_secondary_index_spec.ts | 111 ++++++++++-------- .../__test__/local_secondary_index_spec.ts | 19 +++ 3 files changed, 78 insertions(+), 56 deletions(-) diff --git a/src/query/__test__/full_primary_key_spec.ts b/src/query/__test__/full_primary_key_spec.ts index 0854fea..e5032c3 100644 --- a/src/query/__test__/full_primary_key_spec.ts +++ b/src/query/__test__/full_primary_key_spec.ts @@ -237,10 +237,6 @@ describe("FullPrimaryKey", () => { const res = await primaryKey.scanAll({}); expect(res.records.length).to.eq(3); - // Ordered by range key since it's "scan" - expect(res.records[0].title).to.eq("aba"); - expect(res.records[1].title).to.eq("abc"); - expect(res.records[2].title).to.eq("abd"); }); it("should find all items with parallelize", async () => { diff --git a/src/query/__test__/global_secondary_index_spec.ts b/src/query/__test__/global_secondary_index_spec.ts index c5c9f3b..5f1dbf1 100644 --- a/src/query/__test__/global_secondary_index_spec.ts +++ b/src/query/__test__/global_secondary_index_spec.ts @@ -7,6 +7,14 @@ import { Table } from '../../table'; @Decorator.Table({ name: "prod-Card" }) class Card extends Table { + static create(id: number, title: string, count: number) { + const record = new this(); + record.id = id; + record.title = title; + record.count = count; + return record; + } + @Decorator.Attribute() public id: number; @@ -24,6 +32,9 @@ class Card extends Table { @Decorator.FullGlobalSecondaryIndex('title', 'id') static readonly fullTitleIndex: Query.FullGlobalSecondaryIndex; + + @Decorator.Writer() + static readonly writer: Query.Writer; } describe("HashGlobalSecondaryIndex", () => { @@ -37,28 +48,11 @@ describe("HashGlobalSecondaryIndex", () => { describe("#query", () => { it("should find items", async () => { - await Card.metadata.connection.documentClient.put({ - TableName: Card.metadata.name, - Item: { - id: 10, - title: "abc", - }, - }).promise(); - await Card.metadata.connection.documentClient.put({ - TableName: Card.metadata.name, - Item: { - id: 11, - title: "abd", - }, - }).promise(); - await Card.metadata.connection.documentClient.put({ - TableName: Card.metadata.name, - Item: { - id: 12, - title: "abd", - }, - }).promise(); - + await Card.writer.batchPut([ + Card.create(10, "abc", 1), + Card.create(11, "abd", 1), + Card.create(12, "abd", 1), + ]); const res = await Card.hashTitleIndex.query("abd"); expect(res.records.length).to.eq(2); @@ -66,6 +60,25 @@ describe("HashGlobalSecondaryIndex", () => { expect(res.records[1].id).to.eq(11); }); }); + + describe("#scanAll", () => { + it("should find all items", async () => { + await Card.writer.batchPut([ + Card.create(10, "abc", 1), + Card.create(11, "abd", 1), + Card.create(12, "abd", 1), + Card.create(13, "abd", 1), + Card.create(14, "abe", 1), + Card.create(15, "abe", 1), + ]); + const res = await Card.hashTitleIndex.scanAll({ + scanBatchSize: 1, + parallelize: 3, + }); + + expect(res.records.length).to.eq(6); + }); + }); }); describe("FullGlobalSecondaryIndex", () => { @@ -79,36 +92,12 @@ describe("FullGlobalSecondaryIndex", () => { describe("#query", () => { it("should find items", async () => { - await Card.metadata.connection.documentClient.put({ - TableName: Card.metadata.name, - Item: { - id: 10, - title: "abc", - }, - }).promise(); - await Card.metadata.connection.documentClient.put({ - TableName: Card.metadata.name, - Item: { - id: 11, - title: "abd", - }, - }).promise(); - await Card.metadata.connection.documentClient.put({ - TableName: Card.metadata.name, - Item: { - id: 12, - title: "abd", - }, - }).promise(); - await Card.metadata.connection.documentClient.put({ - TableName: Card.metadata.name, - Item: { - id: 13, - title: "abd", - }, - }).promise(); - - + await Card.writer.batchPut([ + Card.create(10, "abc", 1), + Card.create(11, "abd", 1), + Card.create(12, "abd", 1), + Card.create(13, "abd", 1), + ]); const res = await Card.fullTitleIndex.query({ hash: "abd", range: [">=", 12], @@ -120,4 +109,22 @@ describe("FullGlobalSecondaryIndex", () => { expect(res.records[1].id).to.eq(12); }); }); + + describe("#scanAll", () => { + it("should find all items", async () => { + await Card.writer.batchPut([ + Card.create(10, "abc", 1), + Card.create(11, "abd", 1), + Card.create(12, "abd", 1), + Card.create(13, "abd", 1), + Card.create(14, "abe", 1), + Card.create(15, "abe", 1), + ]); + const res = await Card.fullTitleIndex.scanAll({ + scanBatchSize: 1, + parallelize: 3, + }); + expect(res.records.length).to.eq(6); + }); + }); }); diff --git a/src/query/__test__/local_secondary_index_spec.ts b/src/query/__test__/local_secondary_index_spec.ts index 8baa952..6551ed6 100644 --- a/src/query/__test__/local_secondary_index_spec.ts +++ b/src/query/__test__/local_secondary_index_spec.ts @@ -64,4 +64,23 @@ describe("LocalSecondaryIndex", () => { expect(res.records[1].count).to.eq(3); }); }); + + describe("#scanAll", () => { + it("should find all items", async () => { + await Card.writer.batchPut([ + Card.create(10, "abc", 1), + Card.create(11, "abd", 2), + Card.create(12, "abd", 3), + Card.create(13, "abd", 4), + Card.create(14, "abe", 5), + Card.create(15, "abe", 6), + ]); + const res = await Card.countIndex.scanAll({ + scanBatchSize: 1, + parallelize: 3, + }); + + expect(res.records.length).to.eq(6); + }); + }); });