From cbf391e32e26978d558082021010b6b614d5defa Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Mon, 18 Sep 2023 22:43:50 -0500 Subject: [PATCH] Chore/deprecate PSV/TSV extractors (#226) * Deprecate PSV/TSV extractors * Use specific version * chore: changeset * feat: remove pipe & tab delimiters & add messages * Update good-bobcats-tease.md * chore: update package-lock.json * fix: return listener that console.logs deprecation message --- .changeset/good-bobcats-tease.md | 7 ++++ package-lock.json | 6 +-- .../delimiter-extractor/ref/test-basic.pound | 4 ++ .../delimiter-extractor/ref/test-basic.psv | 4 -- .../ref/test-complex.pound | 13 +++++++ .../delimiter-extractor/ref/test-complex.psv | 13 ------- plugins/delimiter-extractor/src/index.ts | 2 - .../delimiter-extractor/src/parser.spec.ts | 38 +++++++++---------- plugins/psv-extractor/README.md | 8 +--- plugins/psv-extractor/package.json | 3 +- plugins/psv-extractor/src/index.ts | 13 +++++-- plugins/tsv-extractor/README.md | 3 ++ plugins/tsv-extractor/package.json | 3 +- plugins/tsv-extractor/src/index.ts | 11 +++++- 14 files changed, 70 insertions(+), 58 deletions(-) create mode 100644 .changeset/good-bobcats-tease.md create mode 100644 plugins/delimiter-extractor/ref/test-basic.pound delete mode 100644 plugins/delimiter-extractor/ref/test-basic.psv create mode 100644 plugins/delimiter-extractor/ref/test-complex.pound delete mode 100644 plugins/delimiter-extractor/ref/test-complex.psv create mode 100644 plugins/tsv-extractor/README.md diff --git a/.changeset/good-bobcats-tease.md b/.changeset/good-bobcats-tease.md new file mode 100644 index 000000000..3abd6c5c5 --- /dev/null +++ b/.changeset/good-bobcats-tease.md @@ -0,0 +1,7 @@ +--- +'@flatfile/plugin-delimiter-extractor': minor +'@flatfile/plugin-psv-extractor': minor +'@flatfile/plugin-tsv-extractor': minor +--- + +Deprecate PSV/TSV extractors since PSV/TSV files are now natively supported by the Flatfile Platform. diff --git a/package-lock.json b/package-lock.json index c97c4a6b8..99fc48033 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10625,8 +10625,7 @@ "version": "1.5.0", "license": "ISC", "dependencies": { - "@flatfile/api": "^1.5.13", - "@flatfile/plugin-delimiter-extractor": "^0.6.0" + "@flatfile/api": "^1.5.13" }, "engines": { "node": ">= 16" @@ -10673,8 +10672,7 @@ "version": "1.4.0", "license": "ISC", "dependencies": { - "@flatfile/api": "^1.5.13", - "@flatfile/plugin-delimiter-extractor": "^0.6.0" + "@flatfile/api": "^1.5.13" }, "engines": { "node": ">= 16" diff --git a/plugins/delimiter-extractor/ref/test-basic.pound b/plugins/delimiter-extractor/ref/test-basic.pound new file mode 100644 index 000000000..e39874054 --- /dev/null +++ b/plugins/delimiter-extractor/ref/test-basic.pound @@ -0,0 +1,4 @@ +"Code*"#"Details"#"BranchName"#"Tenant" +"Personal Care"#"Personal Care"#"Department"#"notdata" +Hospital#Hospital#Department +Home Nursing#Home Nursing#Department#false \ No newline at end of file diff --git a/plugins/delimiter-extractor/ref/test-basic.psv b/plugins/delimiter-extractor/ref/test-basic.psv deleted file mode 100644 index 056996406..000000000 --- a/plugins/delimiter-extractor/ref/test-basic.psv +++ /dev/null @@ -1,4 +0,0 @@ -"Code*"|"Details"|"BranchName"|"Tenant" -"Personal Care"|"Personal Care"|"Department"|"notdata" -Hospital|Hospital|Department -Home Nursing|Home Nursing|Department|false \ No newline at end of file diff --git a/plugins/delimiter-extractor/ref/test-complex.pound b/plugins/delimiter-extractor/ref/test-complex.pound new file mode 100644 index 000000000..473011aa6 --- /dev/null +++ b/plugins/delimiter-extractor/ref/test-complex.pound @@ -0,0 +1,13 @@ + + + +hello + + + +"Code*"#"Details"#"BranchName"#"Tenant" +"Personal Care"#"Personal Care"#"Department"#"notdata" + + +Hospital#Hospital#Department +Home Nursing#Home Nursing#Department#false \ No newline at end of file diff --git a/plugins/delimiter-extractor/ref/test-complex.psv b/plugins/delimiter-extractor/ref/test-complex.psv deleted file mode 100644 index c2c244503..000000000 --- a/plugins/delimiter-extractor/ref/test-complex.psv +++ /dev/null @@ -1,13 +0,0 @@ - - - -hello - - - -"Code*"|"Details"|"BranchName"|"Tenant" -"Personal Care"|"Personal Care"|"Department"|"notdata" - - -Hospital|Hospital|Department -Home Nursing|Home Nursing|Department|false \ No newline at end of file diff --git a/plugins/delimiter-extractor/src/index.ts b/plugins/delimiter-extractor/src/index.ts index 555c6640c..66573a1bf 100644 --- a/plugins/delimiter-extractor/src/index.ts +++ b/plugins/delimiter-extractor/src/index.ts @@ -3,8 +3,6 @@ import { Extractor } from '@flatfile/util-extractor' import { parseBuffer } from './parser' const delimiterMap = { - '\t': 'tsv', - '|': 'pipe', ',': 'csv', ';': 'ssv', ':': 'dsv', diff --git a/plugins/delimiter-extractor/src/parser.spec.ts b/plugins/delimiter-extractor/src/parser.spec.ts index e51d2c1a4..b8b7a5a03 100644 --- a/plugins/delimiter-extractor/src/parser.spec.ts +++ b/plugins/delimiter-extractor/src/parser.spec.ts @@ -1,18 +1,18 @@ -import { parseBuffer } from './parser' -import * as path from 'path' import * as fs from 'fs' +import * as path from 'path' +import { parseBuffer } from './parser' describe('parser', () => { - const psvBasicBuffer: Buffer = fs.readFileSync( - path.join(__dirname, '../ref/test-basic.psv') + const colonBasicBuffer: Buffer = fs.readFileSync( + path.join(__dirname, '../ref/test-basic.pound') ) - const psvComplexBuffer: Buffer = fs.readFileSync( - path.join(__dirname, '../ref/test-complex.psv') + const colonComplexBuffer: Buffer = fs.readFileSync( + path.join(__dirname, '../ref/test-complex.pound') ) - test('PSV to WorkbookCapture', () => { - expect(parseBuffer(psvBasicBuffer, { delimiter: '|' })).toEqual({ + test('colon to WorkbookCapture', () => { + expect(parseBuffer(colonBasicBuffer, { delimiter: '#' })).toEqual({ Sheet1: { headers: ['Code', 'Details', 'BranchName', 'Tenant'], required: { @@ -44,13 +44,13 @@ describe('parser', () => { }) }) it('has headers', () => { - const headers = parseBuffer(psvBasicBuffer, { delimiter: '|' }).Sheet1 + const headers = parseBuffer(colonBasicBuffer, { delimiter: '#' }).Sheet1 .headers expect(headers).toEqual(['Code', 'Details', 'BranchName', 'Tenant']) }) test('transform', () => { - const data = parseBuffer(psvBasicBuffer, { - delimiter: '|', + const data = parseBuffer(colonBasicBuffer, { + delimiter: '#', transform: (v: string) => { return v.toUpperCase() }, @@ -76,8 +76,8 @@ describe('parser', () => { ]) }) test('dynamicTyping', () => { - const data = parseBuffer(psvBasicBuffer, { - delimiter: '|', + const data = parseBuffer(colonBasicBuffer, { + delimiter: '#', dynamicTyping: true, }).Sheet1.data expect(data).toEqual([ @@ -101,8 +101,8 @@ describe('parser', () => { ]) }) test('skipEmptyLines', () => { - const data = parseBuffer(psvBasicBuffer, { - delimiter: '|', + const data = parseBuffer(colonBasicBuffer, { + delimiter: '#', skipEmptyLines: true, }).Sheet1.data expect(data).toEqual([ @@ -128,13 +128,13 @@ describe('parser', () => { test('empty buffer', () => { const emptyBuffer = Buffer.from('', 'utf8') const logSpy = jest.spyOn(global.console, 'log') - parseBuffer(emptyBuffer, { delimiter: '|' }) + parseBuffer(emptyBuffer, { delimiter: '#' }) expect(logSpy).toHaveBeenCalledWith('No data found in the file') - expect(parseBuffer(emptyBuffer, { delimiter: '|' })).toEqual({}) + expect(parseBuffer(emptyBuffer, { delimiter: '#' })).toEqual({}) }) test('parsing of basic equals parsing of complex', () => { - expect(parseBuffer(psvComplexBuffer, { delimiter: '|' })).toEqual( - parseBuffer(psvBasicBuffer, { delimiter: '|' }) + expect(parseBuffer(colonComplexBuffer, { delimiter: '#' })).toEqual( + parseBuffer(colonBasicBuffer, { delimiter: '#' }) ) }) }) diff --git a/plugins/psv-extractor/README.md b/plugins/psv-extractor/README.md index c49474d7f..08379902b 100644 --- a/plugins/psv-extractor/README.md +++ b/plugins/psv-extractor/README.md @@ -1,9 +1,3 @@ # @flatfile/plugin-psv-extractor -This plugin parses a PSV file and extracts it into Flatfile. - -`npm i @flatfile/plugin-psv-extractor` - -## Get Started - -Follow [this guide](https://flatfile.com/docs/plugins/extractors/psv-extractor) to learn how to use the plugin. \ No newline at end of file +This plugin has been deprecated. PSV extraction is now natively supported by the Flatfile Platform. \ No newline at end of file diff --git a/plugins/psv-extractor/package.json b/plugins/psv-extractor/package.json index d1eab9a2e..84c35a2bc 100644 --- a/plugins/psv-extractor/package.json +++ b/plugins/psv-extractor/package.json @@ -27,7 +27,6 @@ }, "license": "ISC", "dependencies": { - "@flatfile/api": "^1.5.13", - "@flatfile/plugin-delimiter-extractor": "^0.6.0" + "@flatfile/api": "^1.5.13" } } diff --git a/plugins/psv-extractor/src/index.ts b/plugins/psv-extractor/src/index.ts index 77d7de583..1d1c1f64a 100644 --- a/plugins/psv-extractor/src/index.ts +++ b/plugins/psv-extractor/src/index.ts @@ -1,6 +1,9 @@ import { Flatfile } from '@flatfile/api' -import { DelimiterExtractor } from '@flatfile/plugin-delimiter-extractor' +import { FlatfileListener } from '@flatfile/listener' +/* + * @deprecated + */ export const PSVExtractor = (options?: { dynamicTyping?: boolean skipEmptyLines?: boolean | 'greedy' @@ -9,10 +12,14 @@ export const PSVExtractor = (options?: { parallel?: number debug?: boolean }) => { - return DelimiterExtractor('.psv', { delimiter: '|', ...options }) + return (listener: FlatfileListener) => { + console.log( + 'The PSVExtractor plugin is deprecated. PSV extraction is now natively supported by the Flatfile Platform.' + ) + } } /* - * @deprecated use `PSVExtractor` instead + * @deprecated */ export const psvExtractorPlugin = PSVExtractor diff --git a/plugins/tsv-extractor/README.md b/plugins/tsv-extractor/README.md new file mode 100644 index 000000000..bf2a30f4d --- /dev/null +++ b/plugins/tsv-extractor/README.md @@ -0,0 +1,3 @@ +# @flatfile/plugin-tsv-extractor + +This plugin has been deprecated. TSV extraction is now natively supported by the Flatfile Platform. \ No newline at end of file diff --git a/plugins/tsv-extractor/package.json b/plugins/tsv-extractor/package.json index ee082c9f8..f312abcbe 100644 --- a/plugins/tsv-extractor/package.json +++ b/plugins/tsv-extractor/package.json @@ -27,7 +27,6 @@ }, "license": "ISC", "dependencies": { - "@flatfile/api": "^1.5.13", - "@flatfile/plugin-delimiter-extractor": "^0.6.0" + "@flatfile/api": "^1.5.13" } } diff --git a/plugins/tsv-extractor/src/index.ts b/plugins/tsv-extractor/src/index.ts index a103c79b5..13ba71746 100644 --- a/plugins/tsv-extractor/src/index.ts +++ b/plugins/tsv-extractor/src/index.ts @@ -1,6 +1,9 @@ import { Flatfile } from '@flatfile/api' -import { DelimiterExtractor } from '@flatfile/plugin-delimiter-extractor' +import { FlatfileListener } from '@flatfile/listener' +/* + * @deprecated + */ export const TSVExtractor = (options?: { dynamicTyping?: boolean skipEmptyLines?: boolean | 'greedy' @@ -9,5 +12,9 @@ export const TSVExtractor = (options?: { parallel?: number debug?: boolean }) => { - return DelimiterExtractor('.tsv', { delimiter: '\t', ...options }) + return (listener: FlatfileListener) => { + console.log( + 'The TSVExtractor plugin is deprecated. TSV extraction is now natively supported by the Flatfile Platform.' + ) + } }