diff --git a/.changeset/forty-crabs-cross.md b/.changeset/forty-crabs-cross.md new file mode 100644 index 000000000..d3a560278 --- /dev/null +++ b/.changeset/forty-crabs-cross.md @@ -0,0 +1,5 @@ +--- +'@flatfile/util-common': minor +--- + +This release adds more common utilities and fixes a bug in processRecords when updating record values. diff --git a/utils/common/README.md b/utils/common/README.md new file mode 100644 index 000000000..c03af1e06 --- /dev/null +++ b/utils/common/README.md @@ -0,0 +1,3 @@ +# @flatfile/util-common + +This is a package of common utility functions used across the Flatfile Plugins ecosystem. \ No newline at end of file diff --git a/utils/common/src/all.records.ts b/utils/common/src/all.records.ts index ce69164ff..eb472f714 100644 --- a/utils/common/src/all.records.ts +++ b/utils/common/src/all.records.ts @@ -68,8 +68,8 @@ export async function processRecords( ) => R | void | Promise, options: Omit = {} ): Promise { - const totalRecords = await getSheetLength(sheetId) const pageSize = options.pageSize ?? DEFAULT_PAGE_SIZE + const totalRecords = await getSheetLength(sheetId) const totalPageCount = Math.ceil(totalRecords / pageSize) || 1 const results: R[] = [] @@ -77,8 +77,17 @@ export async function processRecords( try { const records = (await getRecordsRaw(sheetId, { ...options, + pageSize, pageNumber, })) as Flatfile.Record_[] + + // Delete updatedAt + records.forEach((record) => + Object.values(record.values).forEach( + (value: Record) => delete value.updatedAt + ) + ) + const result = await callback(records, pageNumber, totalPageCount) if (result !== undefined && result !== null) { results.push(result as R) diff --git a/utils/common/src/async.batch.spec.ts b/utils/common/src/async.batch.spec.ts index 10b362acc..82404f644 100644 --- a/utils/common/src/async.batch.spec.ts +++ b/utils/common/src/async.batch.spec.ts @@ -1,5 +1,5 @@ import { FlatfileEvent } from '@flatfile/listener' -import { asyncBatch } from './async.batch' +import { asyncBatch, chunkify } from './async.batch' describe('asyncBatch', () => { it('should split the array into chunks and call the callback for each chunk', async () => { @@ -112,4 +112,41 @@ describe('asyncBatch', () => { expect(callback).toHaveBeenNthCalledWith(2, [3], undefined) expect(results).toEqual([3, 3]) }) + + describe('chunkify()', () => { + it('chunks an array of 10 items into parts of 2', () => { + const arr = Array.from({ length: 10 }, (_, i) => i + 1) + const chunked = chunkify(arr, 2) + expect(chunked.length).toBe(5) + expect(chunked[0]).toEqual([1, 2]) + expect(chunked[4]).toEqual([9, 10]) + }) + + it('handles an array with length not perfectly divisible by chunk size', () => { + const arr = [1, 2, 3, 4, 5] + const chunked = chunkify(arr, 2) + expect(chunked.length).toBe(3) + expect(chunked[0]).toEqual([1, 2]) + expect(chunked[2]).toEqual([5]) + }) + + it('returns the original array if chunk size is greater than array length', () => { + const arr = [1, 2, 3] + const chunked = chunkify(arr, 5) + expect(chunked.length).toBe(1) + expect(chunked[0]).toEqual([1, 2, 3]) + }) + + it('returns an empty array when chunk size is 0', () => { + const arr = [1, 2, 3] + const chunked = chunkify(arr, 0) + expect(chunked).toEqual([]) + }) + + it('returns an empty array when the input array is empty', () => { + const arr = [] + const chunked = chunkify(arr, 3) + expect(chunked).toEqual([]) + }) + }) }) diff --git a/utils/common/src/async.batch.ts b/utils/common/src/async.batch.ts index dd9523305..c11a0bff4 100644 --- a/utils/common/src/async.batch.ts +++ b/utils/common/src/async.batch.ts @@ -7,10 +7,7 @@ export async function asyncBatch( event?: FlatfileEvent ): Promise { const { chunkSize = 10_000, parallel = 1, debug = false } = options - const chunks = Array.from( - { length: Math.ceil(arr.length / chunkSize) }, - (_, i) => arr.slice(i * chunkSize, i * chunkSize + chunkSize) - ) + const chunks = chunkify(arr, chunkSize) if (debug) { console.log(`${chunks.length} chunks to be processed`) @@ -50,3 +47,13 @@ export async function asyncBatch( return Array.from(results.values()) } + +export function chunkify(arr: T[], chunkSize: number): T[][] { + if (chunkSize <= 0) { + return [] + } + + return Array.from({ length: Math.ceil(arr.length / chunkSize) }, (_, i) => + arr.slice(i * chunkSize, i * chunkSize + chunkSize) + ) +} diff --git a/utils/common/src/delete.records.ts b/utils/common/src/delete.records.ts new file mode 100644 index 000000000..510ff8c75 --- /dev/null +++ b/utils/common/src/delete.records.ts @@ -0,0 +1,24 @@ +import type { Flatfile } from '@flatfile/api' +import api from '@flatfile/api' + +export async function deleteRecords( + sheetId: string, + config: Omit +): Promise { + try { + const { data: sheet } = await api.sheets.get(sheetId) + await api.jobs.create({ + type: 'workbook', + operation: 'delete-records', + trigger: 'immediate', + source: sheet.workbookId, + config: { + ...config, + sheet: sheetId, + }, + }) + } catch (error) { + console.error('Error deleting records:', error) + throw new Error('Error deleting records') + } +} diff --git a/utils/common/src/index.ts b/utils/common/src/index.ts index b79e4a9a8..36495655f 100644 --- a/utils/common/src/index.ts +++ b/utils/common/src/index.ts @@ -1,3 +1,4 @@ export * from './all.records' export * from './async.batch' +export * from './delete.records' export * from './logging.helper'