feat: export delimited zip

export/delimited-zip/README.MD

@@ -0,0 +1,61 @@
+<!-- START_INFOCARD -->
+
+The `@flatfile/plugin-export-delimited-zip` plugin exports data from Flatfile sheets to delimited files (e.g., CSV) and compresses them into a ZIP file. This plugin provides an efficient way to download and package your data from Flatfile workbooks.
+
+**Event Type:** 
+`job:ready`
+
+<!-- END_INFOCARD -->
+
+
+> When embedding Flatfile, this plugin should be deployed in a server-side listener. [Learn more](/docs/orchestration/listeners#listener-types)
+
+
+
+## Parameters
+
+#### `job` - `string` 
+The job name to trigger the export. Default: 'downloadDelimited'
+
+#### `delimiter` - `string`
+The delimiter to use in the exported files. Default: ','
+
+#### `fileExtension` - `string`
+The file extension for the exported files. Default: 'csv'
+
+#### `debug` - `boolean` - (optional)
+Enable debug logging. Default: false
+
+
+
+## API Calls
+
+- `api.jobs.ack`
+- `api.jobs.complete`
+- `api.jobs.fail`
+- `api.sheets.list`
+- `api.records.get`
+- `api.files.upload`
+
+
+
+## Usage
+
+**install**
+```bash
+npm install @flatfile/plugin-export-delimited-zip
+```
+
+**listener.ts**
+```typescript
+import type { FlatfileListener } from '@flatfile/listener'
+import exportDelimitedZip from '@flatfile/plugin-export-delimited-zip'
+
+export default function (listener: FlatfileListener) {
+  listener.use(exportDelimitedZip({
+      job: 'export-delimited-zip',
+      delimiter: ',',
+      fileExtension: 'csv',
+    }))
+}
+```

export/delimited-zip/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

export/delimited-zip/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@flatfile/plugin-export-delimited-zip",
+  "version": "0.0.0",
+  "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/export/delimited-zip",
+  "description": "A Flatfile plugin for exporting Workbooks to delimited files and zipping them together",
+  "registryMetadata": {
+    "category": "export"
+  },
+  "engines": {
+    "node": ">= 16"
+  },
+  "browserslist": [
+    "> 0.5%",
+    "last 2 versions",
+    "not dead"
+  ],
+  "browser": {
+    "./dist/index.js": "./dist/index.browser.js",
+    "./dist/index.mjs": "./dist/index.browser.mjs"
+  },
+  "exports": {
+    "types": "./dist/index.d.ts",
+    "node": {
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "browser": {
+      "require": "./dist/index.browser.js",
+      "import": "./dist/index.browser.mjs"
+    },
+    "default": "./dist/index.mjs"
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "source": "./src/index.ts",
+  "files": [
+    "dist/**"
+  ],
+  "scripts": {
+    "build": "rollup -c",
+    "build:watch": "rollup -c --watch",
+    "build:prod": "NODE_ENV=production rollup -c",
+    "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
+    "test": "jest src/*.spec.ts --detectOpenHandles",
+    "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles",
+    "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles"
+  },
+  "keywords": [
+    "flatfile-plugins",
+    "category-export"
+  ],
+  "author": "Flatfile, Inc.",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/FlatFilers/flatfile-plugins.git",
+    "directory": "export/delimited-zip"
+  },
+  "license": "ISC",
+  "dependencies": {
+    "@flatfile/plugin-job-handler": "^0.6.1",
+    "@flatfile/util-common": "^1.4.1",
+    "adm-zip": "^0.5.16",
+    "csv-stringify": "^6.5.1"
+  },
+  "peerDependencies": {
+    "@flatfile/api": "^1.9.15",
+    "@flatfile/listener": "^1.0.5"
+  },
+  "devDependencies": {
+    "@flatfile/rollup-config": "^0.1.1"
+  }
+}

export/delimited-zip/rollup.config.mjs

@@ -0,0 +1,5 @@
+import { buildConfig } from '@flatfile/rollup-config'
+
+const config = buildConfig({})
+
+export default config

export/delimited-zip/src/export.delimited.zip.plugin.ts

@@ -0,0 +1,202 @@
+import { FlatfileClient } from '@flatfile/api'
+import type { FlatfileListener } from '@flatfile/listener'
+import { jobHandler } from '@flatfile/plugin-job-handler'
+import { processRecords } from '@flatfile/util-common'
+import AdmZip from 'adm-zip'
+import { stringify } from 'csv-stringify/sync'
+import fs from 'fs'
+import os from 'os'
+import path from 'path'
+
+const api = new FlatfileClient()
+
+export interface PluginOptions {
+  job: string
+  delimiter: string
+  fileExtension: string
+  debug?: boolean
+}
+
+export function exportDelimitedZip(options: PluginOptions) {
+  return (listener: FlatfileListener) => {
+    listener.use(
+      jobHandler(`workbook:${options.job}`, async (event, tick) => {
+        const { workbookId, spaceId, environmentId } = event.context
+
+        try {
+          const { data: workbook } = await api.workbooks.get(workbookId)
+
+          await tick(1, `Starting ${workbook.name} export`)
+
+          const { data: sheets } = await api.sheets.list({ workbookId })
+          if (options.debug) {
+            console.log('Sheets retrieved:', sheets)
+          }
+
+          // Get current date-time
+          const dateTime = new Date().toISOString().replace(/[:.]/g, '-')
+
+          // Get the path to the system's temporary directory
+          const tempDir = os.tmpdir()
+
+          // Create a new directory in the system's temporary directory for the delimited files
+          const dir = path.join(tempDir, `${workbook.name}_${dateTime}`)
+          if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir)
+          }
+
+          if (options.debug) {
+            console.log(`Creating zip file in ${dir}`)
+          }
+
+          const zipFile = new AdmZip()
+
+          // For each sheet, create a delimited file and populate it with data
+          const totalSheets = sheets.length
+          for (const [index, sheet] of sheets.entries()) {
+            // Limit sheet name to 31 characters
+            const trimmedSheetName = sheet.name.substring(0, 31)
+
+            if (options.debug) {
+              console.log(`Processing sheet ${trimmedSheetName}`)
+            }
+
+            const {
+              data: { records: records },
+            } = await api.records.get(sheet.id, {
+              pageSize: 1,
+            })
+
+            const header = Object.keys(records[0].values)
+
+            const fileName = `${trimmedSheetName}.${options.fileExtension}`
+            const filePath = `${dir}/${fileName}`
+
+            // Write header only once before processing records
+            const headerContent = stringify([header], {
+              delimiter: options.delimiter,
+            })
+            fs.writeFileSync(filePath, headerContent)
+
+            if (options.debug) {
+              console.log(`Writing header to ${filePath}`)
+            }
+
+            await processRecords(
+              sheet.id,
+              async (records, pageNumber, totalPageCount) => {
+                const rows = records.map((record) =>
+                  header.map((key) => record.values[key].value)
+                )
+
+                const csvContent = stringify(rows, {
+                  delimiter: options.delimiter,
+                })
+
+                // Append the new records to the existing file
+                fs.appendFileSync(filePath, csvContent)
+
+                if (options.debug) {
+                  console.log(
+                    `Writing ${records.length} records to ${filePath} (page ${pageNumber} of ${totalPageCount})`
+                  )
+                }
+
+                // Calculate progress percentage
+                const sheetProgress = (pageNumber + 1) / totalPageCount
+                const sheetWeight = 1 / totalSheets
+                const progress = Math.round(
+                  (index / totalSheets + sheetProgress * sheetWeight) * 80 + 10
+                )
+
+                // Acknowledge job progress
+                await tick(
+                  progress,
+                  `Processed page ${pageNumber + 1} of ${trimmedSheetName}.${options.fileExtension}`
+                )
+              },
+              {
+                pageSize: 5,
+              }
+            )
+
+            zipFile.addFile(fileName, fs.readFileSync(filePath))
+          }
+
+          if (options.debug) {
+            console.log(
+              `Data written to ${options.fileExtension.toUpperCase()} files`
+            )
+          }
+
+          const zipFileName = `${workbook.name}_${dateTime}.zip`
+          const zipFilePath = path.join(tempDir, zipFileName)
+
+          zipFile.writeZip(zipFilePath)
+
+          if (options.debug) {
+            console.log(`Zipped file: ${zipFilePath}`)
+          }
+
+          const { data: file } = await api.files.upload(
+            fs.createReadStream(zipFilePath),
+            {
+              spaceId,
+              environmentId,
+              mode: 'export',
+            }
+          )
+
+          // Cleanup temporary files
+          for (const sheet of workbook.sheets) {
+            const trimmedSheetName = sheet.name.substring(0, 31)
+            const fileName = `${trimmedSheetName}.${options.fileExtension}`
+            const filePath = path.join(dir, fileName)
+
+            try {
+              await fs.promises.unlink(filePath)
+              if (options.debug) {
+                console.log(`Deleted temporary file: ${filePath}`)
+              }
+            } catch (error) {
+              console.warn(
+                `Failed to delete temporary file: ${filePath}`,
+                error
+              )
+            }
+          }
+
+          try {
+            await fs.promises.unlink(zipFilePath)
+            if (options.debug) {
+              console.log(`Deleted temporary ZIP file: ${zipFilePath}`)
+            }
+          } catch (error) {
+            console.warn(
+              `Failed to delete temporary ZIP file: ${zipFilePath}`,
+              error
+            )
+          }
+
+          return {
+            outcome: {
+              message: `Data was exported to ${zipFileName}.`,
+              next: {
+                type: 'files',
+                files: [{ fileId: file.id }],
+              },
+            },
+          }
+        } catch (error) {
+          if (options.debug) {
+            console.error(error)
+          }
+
+          throw new Error(
+            `This job failed probably because it couldn't write to the ${options.fileExtension.toUpperCase()} files, compress them into a ZIP file, or upload it.`
+          )
+        }
+      })
+    )
+  }
+}

export/delimited-zip/src/index.ts

@@ -0,0 +1 @@
+export * from './export.delimited.zip.plugin'

export/pivot-table/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}