diff --git a/utils/common/CHANGELOG.md b/utils/common/CHANGELOG.md index 04997e70f..759d211f2 100644 --- a/utils/common/CHANGELOG.md +++ b/utils/common/CHANGELOG.md @@ -1,5 +1,11 @@ # @flatfile/util-common +## 1.3.6 + +### Patch Changes + +- c23f5aa: Adds a more optimized method for streaming records without fetching the total pages first + ## 1.3.4 ### Patch Changes diff --git a/utils/common/package.json b/utils/common/package.json index fa21bb7a4..c14cde635 100644 --- a/utils/common/package.json +++ b/utils/common/package.json @@ -1,6 +1,6 @@ { "name": "@flatfile/util-common", - "version": "1.3.4", + "version": "1.3.6", "description": "A library containing common utilities and helpers for plugins.", "keywords": [], "author": "Carl Brugger", diff --git a/utils/common/src/simple.records.ts b/utils/common/src/simple.records.ts index 0d216671f..662df1a52 100644 --- a/utils/common/src/simple.records.ts +++ b/utils/common/src/simple.records.ts @@ -24,19 +24,54 @@ export class Simplified { tick?: TickHelper ): Promise { const recordCount = await getSheetLength(sheetId) - const pageCount = Math.ceil(recordCount / PAGE_SIZE) + const pageSize = options.pageSize || PAGE_SIZE + const pageCount = Math.ceil(recordCount / pageSize) const recordPages = await asyncLimitSeries(pageCount, async (i: number) => { await tick?.((i + 1) / pageCount, i + 1, pageCount).catch(console.log) const res = await getRecordsRaw(sheetId, { ...options, pageNumber: i + 1, + pageSize, }) return res.map(Simplified.toSimpleRecord) }) return recordPages.flat(1) } + /** + * Return all records for a sheet by iterating until there are empty pages. + * This is most useful in scenarios where pages are generally small but you want + * to safely handle edge cases. It avoids another count request. + * + * @param sheetId + * @param options + */ + static async getAllRecordsSeries( + sheetId: string, + options: Flatfile.records.GetRecordsRequest = {} + ): Promise { + const pageSize = options.pageSize || PAGE_SIZE + + const recordPages: SimpleRecord[][] = [] + let pageNumber = 1 + while (true) { + const res = await getRecordsRaw(sheetId, { + ...options, + pageSize, + pageNumber, + }) + + recordPages.push(res.map(Simplified.toSimpleRecord)) + pageNumber += 1 + if (res.length < pageSize) { + break + } + } + + return recordPages.flat(1) + } + static async findRecordsLimit( sheetId: string, options: Flatfile.records.GetRecordsRequest,