feat: export delimited zip

.changeset/lemon-bears-drive.md

@@ -0,0 +1,11 @@
+---
+'@flatfile/bundler-config-rollup': minor
+'@flatfile/plugin-export-delimited-zip': minor
+'@flatfile/plugin-record-hook': minor
+'@flatfile/plugin-export-pivot-table': minor
+'@flatfile/plugin-rollout': minor
+---
+
+This release introduces the @flatfile/plugin-export-delimited-zip plugin. This plugin runs on a Workbook-level action to export the Workbook's Sheets as CSV files in a Zip file and uploaded back to Flatfile.
+
+Additionally, this PR updates some documentation.

bundlers/rollup-config/index.mjs

@@ -35,12 +35,16 @@ function commonPlugins(browser, umd = false) {
 
 export function buildConfig({
   external = [],
-  includeNode = true,
-  includeBrowser = true,
-  includeDefinition = true,
-  includeUmd = false,
+  includeNode = false,
+  includeBrowser = false,
+  includeDefinition = false,
+  includeUmd = true,
   umdConfig = { name: undefined, external: [] },
 }) {
+  if (includeUmd && !umdConfig.name) {
+    throw new Error('umdConfig.name is required when includeUmd is true')
+  }
+
   return [
     // Node.js build
     ...(includeNode

export/delimited-zip/README.MD

@@ -0,0 +1,61 @@
+<!-- START_INFOCARD -->
+
+The `@flatfile/plugin-export-delimited-zip` plugin exports data from Flatfile sheets to delimited files (e.g., CSV) and compresses them into a ZIP file. This plugin provides an efficient way to download and package your data from Flatfile workbooks.
+
+**Event Type:** 
+`job:ready`
+
+<!-- END_INFOCARD -->
+
+
+> When embedding Flatfile, this plugin should be deployed in a server-side listener. [Learn more](/docs/orchestration/listeners#listener-types)
+
+
+
+## Parameters
+
+#### `job` - `string` 
+The job name to trigger the export. Default: 'downloadDelimited'
+
+#### `delimiter` - `string`
+The delimiter to use in the exported files. Default: ','
+
+#### `fileExtension` - `string`
+The file extension for the exported files. Default: 'csv'
+
+#### `debug` - `boolean` - (optional)
+Enable debug logging. Default: false
+
+
+
+## API Calls
+
+- `api.jobs.ack`
+- `api.jobs.complete`
+- `api.jobs.fail`
+- `api.sheets.list`
+- `api.records.get`
+- `api.files.upload`
+
+
+
+## Usage
+
+**install**
+```bash
+npm install @flatfile/plugin-export-delimited-zip
+```
+
+**listener.ts**
+```typescript
+import type { FlatfileListener } from '@flatfile/listener'
+import { exportDelimitedZip } from '@flatfile/plugin-export-delimited-zip'
+
+export default function (listener: FlatfileListener) {
+  listener.use(exportDelimitedZip({
+      job: 'export-delimited-zip',
+      delimiter: ',',
+      fileExtension: 'csv',
+    }))
+}
+```

export/delimited-zip/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

export/delimited-zip/package.json

@@ -0,0 +1,81 @@
+{
+  "name": "@flatfile/plugin-export-delimited-zip",
+  "version": "0.0.0",
+  "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/export/delimited-zip",
+  "description": "A Flatfile plugin for exporting Workbooks to delimited files and zipping them together",
+  "registryMetadata": {
+    "category": "export"
+  },
+  "type": "module",
+  "engines": {
+    "node": ">= 16"
+  },
+  "browserslist": [
+    "> 0.5%",
+    "last 2 versions",
+    "not dead"
+  ],
+  "exports": {
+    ".": {
+      "node": {
+        "types": {
+          "import": "./dist/index.d.ts",
+          "require": "./dist/index.d.cts"
+        },
+        "import": "./dist/index.js",
+        "require": "./dist/index.cjs"
+      },
+      "browser": {
+        "types": {
+          "import": "./dist/index.d.ts",
+          "require": "./dist/index.d.cts"
+        },
+        "import": "./dist/index.browser.js",
+        "require": "./dist/index.browser.cjs"
+      },
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "source": "./src/index.ts",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist/**"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "build:watch": "tsup --watch",
+    "build:prod": "NODE_ENV=production tsup",
+    "checks": "tsc --noEmit && attw --pack . && publint .",
+    "lint": "tsc --noEmit",
+    "test": "jest src/*.spec.ts --detectOpenHandles",
+    "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles",
+    "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles"
+  },
+  "keywords": [
+    "flatfile-plugins",
+    "category-export"
+  ],
+  "author": "Flatfile, Inc.",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/FlatFilers/flatfile-plugins.git",
+    "directory": "export/delimited-zip"
+  },
+  "license": "ISC",
+  "dependencies": {
+    "@flatfile/plugin-job-handler": "^0.7.0",
+    "@flatfile/util-common": "^1.5.0",
+    "adm-zip": "^0.5.16",
+    "csv-stringify": "^6.5.1"
+  },
+  "peerDependencies": {
+    "@flatfile/api": "^1.9.19",
+    "@flatfile/listener": "^1.1.0"
+  },
+  "devDependencies": {
+    "@flatfile/bundler-config-tsup": "^0.1.0"
+  }
+}

export/delimited-zip/src/export.delimited.zip.plugin.ts

@@ -0,0 +1,195 @@
+import { FlatfileClient } from '@flatfile/api'
+import { jobHandler } from '@flatfile/plugin-job-handler'
+import { processRecords } from '@flatfile/util-common'
+import AdmZip from 'adm-zip'
+import { stringify } from 'csv-stringify/sync'
+import fs from 'fs'
+import { tmpdir } from 'os'
+import path from 'path'
+
+const api = new FlatfileClient()
+
+export interface PluginOptions {
+  job: string
+  delimiter: string
+  fileExtension: string
+  debug?: boolean
+}
+
+export function exportDelimitedZip(options: PluginOptions) {
+  return jobHandler(`workbook:${options.job}`, async (event, tick) => {
+    const { workbookId, spaceId, environmentId } = event.context
+
+    try {
+      const { data: workbook } = await api.workbooks.get(workbookId)
+
+      await tick(1, `Starting ${workbook.name} export`)
+
+      const { data: sheets } = await api.sheets.list({ workbookId })
+      if (options.debug) {
+        console.log('Sheets retrieved:', sheets)
+      }
+
+      // Get current date-time
+      const dateTime = new Date().toISOString().replace(/[:.]/g, '-')
+
+      // Get the path to the system's temporary directory
+      const tempDir = tmpdir()
+
+      // Create a new directory in the system's temporary directory for the delimited files
+      const dir = path.join(tempDir, `${workbook.name}_${dateTime}`)
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir)
+      }
+
+      if (options.debug) {
+        console.log(`Creating zip file in ${dir}`)
+      }
+
+      const zipFile = new AdmZip()
+
+      // For each sheet, create a delimited file and populate it with data
+      const totalSheets = sheets.length
+      for (const [index, sheet] of sheets.entries()) {
+        // Limit sheet name to 31 characters
+        const trimmedSheetName = sheet.name.substring(0, 31)
+
+        if (options.debug) {
+          console.log(`Processing sheet ${trimmedSheetName}`)
+        }
+
+        const {
+          data: {
+            config: { fields },
+          },
+        } = await api.sheets.get(sheet.id)
+
+        const header = fields.map((field) => field.key)
+        const headerLabels = fields.map((field) => field.label)
+
+        const fileName = `${trimmedSheetName}.${options.fileExtension}`
+        const filePath = `${dir}/${fileName}`
+
+        // Write header only once before processing records
+        const headerContent = stringify([headerLabels], {
+          delimiter: options.delimiter,
+        })
+        fs.writeFileSync(filePath, headerContent)
+
+        if (options.debug) {
+          console.log(`Writing header to ${filePath}`)
+        }
+
+        await processRecords(
+          sheet.id,
+          async (records, pageNumber, totalPageCount) => {
+            const rows = records.map((record) =>
+              header.map((key) => record.values[key].value)
+            )
+
+            const csvContent = stringify(rows, {
+              delimiter: options.delimiter,
+            })
+
+            // Append the new records to the existing file
+            fs.appendFileSync(filePath, csvContent)
+
+            if (options.debug) {
+              console.log(
+                `Writing ${records.length} records to ${filePath} (page ${pageNumber} of ${totalPageCount})`
+              )
+            }
+
+            // Calculate progress percentage
+            const sheetProgress = (pageNumber + 1) / totalPageCount
+            const sheetWeight = 1 / totalSheets
+            const progress = Math.round(
+              (index / totalSheets + sheetProgress * sheetWeight) * 80 + 10
+            )
+
+            // Acknowledge job progress
+            await tick(
+              progress,
+              `Processed page ${pageNumber + 1} of ${trimmedSheetName}.${options.fileExtension}`
+            )
+          },
+          {
+            pageSize: 5,
+          }
+        )
+
+        zipFile.addFile(fileName, fs.readFileSync(filePath))
+      }
+
+      if (options.debug) {
+        console.log(
+          `Data written to ${options.fileExtension.toUpperCase()} files`
+        )
+      }
+
+      const zipFileName = `${workbook.name}_${dateTime}.zip`
+      const zipFilePath = path.join(tempDir, zipFileName)
+
+      zipFile.writeZip(zipFilePath)
+
+      if (options.debug) {
+        console.log(`Zipped file: ${zipFilePath}`)
+      }
+
+      const { data: file } = await api.files.upload(
+        fs.createReadStream(zipFilePath),
+        {
+          spaceId,
+          environmentId,
+          mode: 'export',
+        }
+      )
+
+      // Cleanup temporary files
+      for (const sheet of workbook.sheets) {
+        const trimmedSheetName = sheet.name.substring(0, 31)
+        const fileName = `${trimmedSheetName}.${options.fileExtension}`
+        const filePath = path.join(dir, fileName)
+
+        try {
+          await fs.promises.unlink(filePath)
+          if (options.debug) {
+            console.log(`Deleted temporary file: ${filePath}`)
+          }
+        } catch (error) {
+          console.warn(`Failed to delete temporary file: ${filePath}`, error)
+        }
+      }
+
+      try {
+        await fs.promises.unlink(zipFilePath)
+        if (options.debug) {
+          console.log(`Deleted temporary ZIP file: ${zipFilePath}`)
+        }
+      } catch (error) {
+        console.warn(
+          `Failed to delete temporary ZIP file: ${zipFilePath}`,
+          error
+        )
+      }
+
+      return {
+        outcome: {
+          message: `Data was exported to ${zipFileName}.`,
+          next: {
+            type: 'files',
+            files: [{ fileId: file.id }],
+          },
+        },
+      }
+    } catch (error) {
+      if (options.debug) {
+        console.error(error)
+      }
+
+      throw new Error(
+        `This job failed probably because it couldn't write to the ${options.fileExtension.toUpperCase()} files, compress them into a ZIP file, or upload it.`
+      )
+    }
+  })
+}

export/delimited-zip/src/index.ts

@@ -0,0 +1 @@
+export * from './export.delimited.zip.plugin'

export/delimited-zip/tsup.config.mjs

@@ -0,0 +1,3 @@
+import { defineConfig } from '@flatfile/bundler-config-tsup'
+
+export default defineConfig({ includeBrowser: false })