feat: update to common utils (#460)

.changeset/forty-crabs-cross.md

@@ -0,0 +1,5 @@
+---
+'@flatfile/util-common': minor
+---
+
+This release adds more common utilities and fixes a bug in processRecords when updating record values.

utils/common/README.md

@@ -0,0 +1,3 @@
+# @flatfile/util-common
+
+This is a package of common utility functions used across the Flatfile Plugins ecosystem.

utils/common/src/all.records.ts

@@ -68,17 +68,26 @@ export async function processRecords<R>(
   ) => R | void | Promise<R | void>,
   options: Omit<Flatfile.records.GetRecordsRequest, 'pageNumber'> = {}
 ): Promise<R[] | void> {
-  const totalRecords = await getSheetLength(sheetId)
   const pageSize = options.pageSize ?? DEFAULT_PAGE_SIZE
+  const totalRecords = await getSheetLength(sheetId)
   const totalPageCount = Math.ceil(totalRecords / pageSize) || 1
   const results: R[] = []
 
   for (let pageNumber = 1; pageNumber <= totalPageCount; pageNumber++) {
     try {
       const records = (await getRecordsRaw(sheetId, {
         ...options,
+        pageSize,
         pageNumber,
       })) as Flatfile.Record_[]
+
+      // Delete updatedAt
+      records.forEach((record) =>
+        Object.values(record.values).forEach(
+          (value: Record<string, Flatfile.CellValue>) => delete value.updatedAt
+        )
+      )
+
       const result = await callback(records, pageNumber, totalPageCount)
       if (result !== undefined && result !== null) {
         results.push(result as R)

utils/common/src/async.batch.spec.ts

@@ -1,5 +1,5 @@
 import { FlatfileEvent } from '@flatfile/listener'
-import { asyncBatch } from './async.batch'
+import { asyncBatch, chunkify } from './async.batch'
 
 describe('asyncBatch', () => {
   it('should split the array into chunks and call the callback for each chunk', async () => {
@@ -112,4 +112,41 @@ describe('asyncBatch', () => {
     expect(callback).toHaveBeenNthCalledWith(2, [3], undefined)
     expect(results).toEqual([3, 3])
   })
+
+  describe('chunkify()', () => {
+    it('chunks an array of 10 items into parts of 2', () => {
+      const arr = Array.from({ length: 10 }, (_, i) => i + 1)
+      const chunked = chunkify(arr, 2)
+      expect(chunked.length).toBe(5)
+      expect(chunked[0]).toEqual([1, 2])
+      expect(chunked[4]).toEqual([9, 10])
+    })
+
+    it('handles an array with length not perfectly divisible by chunk size', () => {
+      const arr = [1, 2, 3, 4, 5]
+      const chunked = chunkify(arr, 2)
+      expect(chunked.length).toBe(3)
+      expect(chunked[0]).toEqual([1, 2])
+      expect(chunked[2]).toEqual([5])
+    })
+
+    it('returns the original array if chunk size is greater than array length', () => {
+      const arr = [1, 2, 3]
+      const chunked = chunkify(arr, 5)
+      expect(chunked.length).toBe(1)
+      expect(chunked[0]).toEqual([1, 2, 3])
+    })
+
+    it('returns an empty array when chunk size is 0', () => {
+      const arr = [1, 2, 3]
+      const chunked = chunkify(arr, 0)
+      expect(chunked).toEqual([])
+    })
+
+    it('returns an empty array when the input array is empty', () => {
+      const arr = []
+      const chunked = chunkify(arr, 3)
+      expect(chunked).toEqual([])
+    })
+  })
 })

utils/common/src/async.batch.ts

@@ -7,10 +7,7 @@ export async function asyncBatch<T, R>(
   event?: FlatfileEvent
 ): Promise<R[]> {
   const { chunkSize = 10_000, parallel = 1, debug = false } = options
-  const chunks = Array.from(
-    { length: Math.ceil(arr.length / chunkSize) },
-    (_, i) => arr.slice(i * chunkSize, i * chunkSize + chunkSize)
-  )
+  const chunks = chunkify<T>(arr, chunkSize)
 
   if (debug) {
     console.log(`${chunks.length} chunks to be processed`)
@@ -50,3 +47,13 @@ export async function asyncBatch<T, R>(
 
   return Array.from(results.values())
 }
+
+export function chunkify<T>(arr: T[], chunkSize: number): T[][] {
+  if (chunkSize <= 0) {
+    return []
+  }
+
+  return Array.from({ length: Math.ceil(arr.length / chunkSize) }, (_, i) =>
+    arr.slice(i * chunkSize, i * chunkSize + chunkSize)
+  )
+}

utils/common/src/delete.records.ts

@@ -0,0 +1,24 @@
+import type { Flatfile } from '@flatfile/api'
+import api from '@flatfile/api'
+
+export async function deleteRecords(
+  sheetId: string,
+  config: Omit<Flatfile.DeleteRecordsJobConfig, 'sheet'>
+): Promise<void> {
+  try {
+    const { data: sheet } = await api.sheets.get(sheetId)
+    await api.jobs.create({
+      type: 'workbook',
+      operation: 'delete-records',
+      trigger: 'immediate',
+      source: sheet.workbookId,
+      config: {
+        ...config,
+        sheet: sheetId,
+      },
+    })
+  } catch (error) {
+    console.error('Error deleting records:', error)
+    throw new Error('Error deleting records')
+  }
+}

utils/common/src/index.ts

@@ -1,3 +1,4 @@
 export * from './all.records'
 export * from './async.batch'
+export * from './delete.records'
 export * from './logging.helper'