Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ScanAll method at FullPrimaryKey, HashPrimaryKey #11

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

ScanAll method at FullPrimaryKey, HashPrimaryKey #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
076339e
Add scanAll at fullPrimaryKey
extJo Jul 8, 2020
a00a6d9
Add scanAll at hashPrimaryKey
extJo Jul 8, 2020
d5e40e0
Seperate scanAll for common function
extJo Jul 9, 2020
3ba2bb5
change test logic
extJo Jul 9, 2020
8871e51
change scan all logic
extJo Jul 9, 2020
fff2ba8
Remove throw error logic
extJo Jul 9, 2020
9045b36
Add test
extJo Jul 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions src/query/__test__/full_primary_key_spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,77 @@ describe("FullPrimaryKey", () => {
expect(res.records[1].title).to.eq("abc");
});
});

describe("#scanAll", () => {
it("should find all items", async () => {
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 10,
title: "abc",
},
}).promise();
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 10,
title: "abd",
},
}).promise();
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 10,
title: "aba",
},
}).promise();

const res = await primaryKey.scanAll({});

expect(res.records.length).to.eq(3);
});

it("should find all items with parallelize", async () => {
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 10,
title: "abc",
},
}).promise();
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 10,
title: "abd",
},
}).promise();
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 10,
title: "aba",
},
}).promise();
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 10,
title: "ccc",
},
}).promise();

const res = await primaryKey.scanAll({ parallelize: 3 });

expect(res.records.length).to.eq(4);
// Ordered by range key since it's "scan"
expect(res.records[0].title).to.eq("aba");
expect(res.records[1].title).to.eq("abc");
expect(res.records[2].title).to.eq("abd");
expect(res.records[3].title).to.eq("ccc");
});
});

describe("#update", () => {
it("should be able to update items", async () => {
await primaryKey.update(10, "abc", { count: ["ADD", 1] });
Expand Down
111 changes: 59 additions & 52 deletions src/query/__test__/global_secondary_index_spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ import { Table } from '../../table';

@Decorator.Table({ name: "prod-Card" })
class Card extends Table {
static create(id: number, title: string, count: number) {
const record = new this();
record.id = id;
record.title = title;
record.count = count;
return record;
}

@Decorator.Attribute()
public id: number;

Expand All @@ -24,6 +32,9 @@ class Card extends Table {

@Decorator.FullGlobalSecondaryIndex('title', 'id')
static readonly fullTitleIndex: Query.FullGlobalSecondaryIndex<Card, string, number>;

@Decorator.Writer()
static readonly writer: Query.Writer<Card>;
}

describe("HashGlobalSecondaryIndex", () => {
Expand All @@ -37,35 +48,37 @@ describe("HashGlobalSecondaryIndex", () => {

describe("#query", () => {
it("should find items", async () => {
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 10,
title: "abc",
},
}).promise();
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 11,
title: "abd",
},
}).promise();
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 12,
title: "abd",
},
}).promise();

await Card.writer.batchPut([
Card.create(10, "abc", 1),
Card.create(11, "abd", 1),
Card.create(12, "abd", 1),
]);

const res = await Card.hashTitleIndex.query("abd");
expect(res.records.length).to.eq(2);
expect(res.records[0].id).to.eq(12);
expect(res.records[1].id).to.eq(11);
});
});

describe("#scanAll", () => {
it("should find all items", async () => {
await Card.writer.batchPut([
Card.create(10, "abc", 1),
Card.create(11, "abd", 1),
Card.create(12, "abd", 1),
Card.create(13, "abd", 1),
Card.create(14, "abe", 1),
Card.create(15, "abe", 1),
]);
const res = await Card.hashTitleIndex.scanAll({
scanBatchSize: 1,
parallelize: 3,
});

expect(res.records.length).to.eq(6);
});
});
});

describe("FullGlobalSecondaryIndex", () => {
Expand All @@ -79,36 +92,12 @@ describe("FullGlobalSecondaryIndex", () => {

describe("#query", () => {
it("should find items", async () => {
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 10,
title: "abc",
},
}).promise();
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 11,
title: "abd",
},
}).promise();
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 12,
title: "abd",
},
}).promise();
await Card.metadata.connection.documentClient.put({
TableName: Card.metadata.name,
Item: {
id: 13,
title: "abd",
},
}).promise();


await Card.writer.batchPut([
Card.create(10, "abc", 1),
Card.create(11, "abd", 1),
Card.create(12, "abd", 1),
Card.create(13, "abd", 1),
]);
const res = await Card.fullTitleIndex.query({
hash: "abd",
range: [">=", 12],
Expand All @@ -120,4 +109,22 @@ describe("FullGlobalSecondaryIndex", () => {
expect(res.records[1].id).to.eq(12);
});
});

describe("#scanAll", () => {
it("should find all items", async () => {
await Card.writer.batchPut([
Card.create(10, "abc", 1),
Card.create(11, "abd", 1),
Card.create(12, "abd", 1),
Card.create(13, "abd", 1),
Card.create(14, "abe", 1),
Card.create(15, "abe", 1),
]);
const res = await Card.fullTitleIndex.scanAll({
scanBatchSize: 1,
parallelize: 3,
});
expect(res.records.length).to.eq(6);
});
});
});
58 changes: 58 additions & 0 deletions src/query/__test__/hash_primary_key_spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,64 @@ describe("HashPrimaryKey", () => {
});
});

describe("#scanAll", () => {
it("should find all items", async () => {
async function createCard() {
const card = new Card();
card.id = faker.random.number();
await card.save();
return card;
}

const cards = [
await createCard(),
await createCard(),
await createCard(),
await createCard(),
await createCard(),
await createCard(),
await createCard(),
await createCard(),
await createCard(),
await createCard(),
await createCard(),
];

const res = await primaryKey.scanAll({ scanBatchSize: 1 });

const ids = _.sortBy(
res.records,
(item) => item.id,
).map((c) => c.id);

expect(ids).to.deep.eq( _.sortBy(cards.map((c) => c.id), (i) => i) );
});

it("should find all items with parallelize", async () => {
async function createCard() {
const card = new Card();
card.id = faker.random.number();
await card.save();
return card;
}

const cards = [
await createCard(),
await createCard(),
await createCard(),
await createCard(),
];

const res = await primaryKey.scanAll({ parallelize: 3 });

const ids = _.sortBy(
res.records,
(item) => item.id,
).map((c) => c.id);

expect(ids).to.deep.eq( _.sortBy(cards.map((c) => c.id), (i) => i) );
});
});

describe("#batchGet", async () => {
it ("should return results in order", async () => {
Expand Down
19 changes: 19 additions & 0 deletions src/query/__test__/local_secondary_index_spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,23 @@ describe("LocalSecondaryIndex", () => {
expect(res.records[1].count).to.eq(3);
});
});

describe("#scanAll", () => {
it("should find all items", async () => {
await Card.writer.batchPut([
Card.create(10, "abc", 1),
Card.create(11, "abd", 2),
Card.create(12, "abd", 3),
Card.create(13, "abd", 4),
Card.create(14, "abe", 5),
Card.create(15, "abe", 6),
]);
const res = await Card.countIndex.scanAll({
scanBatchSize: 1,
parallelize: 3,
});

expect(res.records.length).to.eq(6);
});
});
});
17 changes: 17 additions & 0 deletions src/query/full_primary_key.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import * as Query from './query';

import { batchGetFull, batchGetTrim } from "./batch_get";
import { batchWrite } from "./batch_write";
import { scanAll } from "./scan_all";

const HASH_KEY_REF = "#hk";
const HASH_VALUE_REF = ":hkv";
Expand Down Expand Up @@ -186,6 +187,22 @@ export class FullPrimaryKey<T extends Table, HashKeyType, RangeKeyType> {
};
}

async scanAll(options: {
parallelize?: number;
scanBatchSize?: number;
}) {
const res = await scanAll(
this.tableClass.metadata.connection.documentClient,
this.tableClass.metadata.name,
options,
);

return {
records: res.map((item) => Codec.deserialize(this.tableClass, item)),
count: res.length,
};
}

async update(
hashKey: HashKeyType,
sortKey: RangeKeyType,
Expand Down
Loading