Add streaming helper (#547)

* feat: add streaming helper to common utilities * chore: changeset * chore: change apply * release
FlatFilers · Jun 29, 2024 · ba7105b · ba7105b
1 parent d22b9ee
commit ba7105b
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 2 deletions.
diff --git a/utils/common/CHANGELOG.md b/utils/common/CHANGELOG.md
@@ -1,5 +1,11 @@
 # @flatfile/util-common
 
+## 1.3.6
+
+### Patch Changes
+
+- c23f5aa: Adds a more optimized method for streaming records without fetching the total pages first
+
 ## 1.3.4
 
 ### Patch Changes

diff --git a/utils/common/package.json b/utils/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@flatfile/util-common",
-  "version": "1.3.4",
+  "version": "1.3.6",
   "description": "A library containing common utilities and helpers for plugins.",
   "keywords": [],
   "author": "Carl Brugger",

diff --git a/utils/common/src/simple.records.ts b/utils/common/src/simple.records.ts
@@ -24,19 +24,54 @@ export class Simplified {
     tick?: TickHelper
   ): Promise<SimpleRecord[]> {
     const recordCount = await getSheetLength(sheetId)
-    const pageCount = Math.ceil(recordCount / PAGE_SIZE)
+    const pageSize = options.pageSize || PAGE_SIZE
+    const pageCount = Math.ceil(recordCount / pageSize)
 
     const recordPages = await asyncLimitSeries(pageCount, async (i: number) => {
       await tick?.((i + 1) / pageCount, i + 1, pageCount).catch(console.log)
       const res = await getRecordsRaw(sheetId, {
         ...options,
         pageNumber: i + 1,
+        pageSize,
       })
       return res.map(Simplified.toSimpleRecord)
     })
     return recordPages.flat(1)
   }
 
+  /**
+   * Return all records for a sheet by iterating until there are empty pages.
+   * This is most useful in scenarios where pages are generally small but you want
+   * to safely handle edge cases. It avoids another count request.
+   *
+   * @param sheetId
+   * @param options
+   */
+  static async getAllRecordsSeries(
+    sheetId: string,
+    options: Flatfile.records.GetRecordsRequest = {}
+  ): Promise<SimpleRecord[]> {
+    const pageSize = options.pageSize || PAGE_SIZE
+
+    const recordPages: SimpleRecord[][] = []
+    let pageNumber = 1
+    while (true) {
+      const res = await getRecordsRaw(sheetId, {
+        ...options,
+        pageSize,
+        pageNumber,
+      })
+
+      recordPages.push(res.map(Simplified.toSimpleRecord))
+      pageNumber += 1
+      if (res.length < pageSize) {
+        break
+      }
+    }
+
+    return recordPages.flat(1)
+  }
+
   static async findRecordsLimit(
     sheetId: string,
     options: Flatfile.records.GetRecordsRequest,