diff --git a/packages/dsv/package.json b/packages/dsv/package.json index 86b826d19..becead637 100755 --- a/packages/dsv/package.json +++ b/packages/dsv/package.json @@ -52,6 +52,7 @@ "@rollup/pluginutils": "^5.0.1", "@types/d3-dsv": "^3.0.0", "d3-dsv": "2.0.0", + "strip-bom": "^4.0.0", "tosource": "^2.0.0-alpha.3" }, "devDependencies": { diff --git a/packages/dsv/src/index.js b/packages/dsv/src/index.js index 1612d1020..ec94aa446 100755 --- a/packages/dsv/src/index.js +++ b/packages/dsv/src/index.js @@ -3,6 +3,7 @@ import { extname } from 'path'; import { csvParse, tsvParse } from 'd3-dsv'; import toSource from 'tosource'; import { createFilter } from '@rollup/pluginutils'; +import stripBom from 'strip-bom'; const parsers = { '.csv': csvParse, '.tsv': tsvParse }; @@ -18,7 +19,7 @@ export default function dsv(options = {}) { const ext = extname(id); if (!(ext in parsers)) return null; - let rows = parsers[ext](code); + let rows = parsers[ext](stripBom(code)); if (options.processRow) { rows = rows.map((row) => options.processRow(row, id) || row); diff --git a/packages/dsv/test/fixtures/csv-with-bom/fruit.csv b/packages/dsv/test/fixtures/csv-with-bom/fruit.csv new file mode 100755 index 000000000..fc407b09a --- /dev/null +++ b/packages/dsv/test/fixtures/csv-with-bom/fruit.csv @@ -0,0 +1,4 @@ +type,count +apples,7 +pears,4 +bananas,5 diff --git a/packages/dsv/test/fixtures/csv-with-bom/main.js b/packages/dsv/test/fixtures/csv-with-bom/main.js new file mode 100755 index 000000000..a1013c6f0 --- /dev/null +++ b/packages/dsv/test/fixtures/csv-with-bom/main.js @@ -0,0 +1,7 @@ +import fruit from './fruit.csv'; + +t.deepEqual(fruit, [ + { type: 'apples', count: '7' }, + { type: 'pears', count: '4' }, + { type: 'bananas', count: '5' } +]); diff --git a/packages/dsv/test/fixtures/tsv-with-bom/fruit.tsv b/packages/dsv/test/fixtures/tsv-with-bom/fruit.tsv new file mode 100755 index 000000000..3fa3efc34 --- /dev/null +++ b/packages/dsv/test/fixtures/tsv-with-bom/fruit.tsv @@ -0,0 +1,4 @@ +type count +apples 7 +pears 4 +bananas 5 diff --git a/packages/dsv/test/fixtures/tsv-with-bom/main.js b/packages/dsv/test/fixtures/tsv-with-bom/main.js new file mode 100755 index 000000000..e9490cf0e --- /dev/null +++ b/packages/dsv/test/fixtures/tsv-with-bom/main.js @@ -0,0 +1,7 @@ +import fruit from './fruit.tsv'; + +t.deepEqual(fruit, [ + { type: 'apples', count: '7' }, + { type: 'pears', count: '4' }, + { type: 'bananas', count: '5' } +]); diff --git a/packages/dsv/test/test.js b/packages/dsv/test/test.js index c053bceab..d5a59715f 100755 --- a/packages/dsv/test/test.js +++ b/packages/dsv/test/test.js @@ -22,6 +22,15 @@ test('converts a csv file', async (t) => { return testBundle(t, bundle); }); +test('converts a csv file with bom', async (t) => { + const bundle = await rollup({ + input: 'fixtures/csv-with-bom/main.js', + plugins: [dsv()] + }); + t.plan(1); + return testBundle(t, bundle); +}); + test('converts a tsv file', async (t) => { const bundle = await rollup({ input: 'fixtures/basic-tsv/main.js', @@ -31,6 +40,15 @@ test('converts a tsv file', async (t) => { return testBundle(t, bundle); }); +test('converts a tsv file with bom', async (t) => { + const bundle = await rollup({ + input: 'fixtures/tsv-with-bom/main.js', + plugins: [dsv()] + }); + t.plan(1); + return testBundle(t, bundle); +}); + test('uses a custom processor', async (t) => { const parse = (value) => (isNaN(+value) ? value : +value); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 17b8e2cef..767e65c89 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -267,6 +267,9 @@ importers: d3-dsv: specifier: 2.0.0 version: 2.0.0 + strip-bom: + specifier: ^4.0.0 + version: 4.0.0 tosource: specifier: ^2.0.0-alpha.3 version: 2.0.0-alpha.3 @@ -2496,6 +2499,7 @@ packages: resolution: {integrity: sha512-BGnRktAZq4RI6FSicI+F6ws9paiYmjyaXUNKSukLthzgzPC91V4SXVylbFOCKvrhdWAr0lvZgcTrkgYNAmAcuQ==} cpu: [arm64] os: [linux] + libc: [glibc] requiresBuild: true optional: true @@ -2503,6 +2507,7 @@ packages: resolution: {integrity: sha512-FzhHpp+vRTjIUYXMExj9Ffj2bCQgnRAzlWlsQTdYGYvPQMVadfPMvnlcr4Li8P7Yv1iBFtDzRVfZAgL5glvIAA==} cpu: [x64] os: [linux] + libc: [glibc] requiresBuild: true optional: true @@ -2510,6 +2515,7 @@ packages: resolution: {integrity: sha512-0y+oXnCCT5+U5V58bY7dy65yDrWWfopFJwtC2EbFeA9SHrVjG36/TQo535ML3zdFwO+fma8r5FP1os0psbQBXQ==} cpu: [x64] os: [linux] + libc: [musl] requiresBuild: true optional: true @@ -2595,6 +2601,7 @@ packages: engines: {node: '>=10'} cpu: [arm64] os: [linux] + libc: [glibc] requiresBuild: true dev: true optional: true @@ -2604,6 +2611,7 @@ packages: engines: {node: '>=10'} cpu: [arm64] os: [linux] + libc: [musl] requiresBuild: true dev: true optional: true @@ -2613,6 +2621,7 @@ packages: engines: {node: '>=10'} cpu: [x64] os: [linux] + libc: [glibc] requiresBuild: true dev: true optional: true @@ -2622,6 +2631,7 @@ packages: engines: {node: '>=10'} cpu: [x64] os: [linux] + libc: [musl] requiresBuild: true dev: true optional: true @@ -7511,7 +7521,6 @@ packages: /strip-bom@4.0.0: resolution: {integrity: sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==} engines: {node: '>=8'} - dev: true /strip-final-newline@2.0.0: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==}