fix(dsv): strip bom from csv/tsv files if necessary (#1745)

* fix(dsv): strip bom from csv/tsv files if necessary

* resolve conflicts in pnpm-lock.yaml

* update pnpm-lock.yaml

---------

Co-authored-by: shellscape <[email protected]>

packages/dsv/package.json

@@ -52,6 +52,7 @@
     "@rollup/pluginutils": "^5.0.1",
     "@types/d3-dsv": "^3.0.0",
     "d3-dsv": "2.0.0",
+    "strip-bom": "^4.0.0",
     "tosource": "^2.0.0-alpha.3"
   },
   "devDependencies": {

packages/dsv/src/index.js

@@ -3,6 +3,7 @@ import { extname } from 'path';
 import { csvParse, tsvParse } from 'd3-dsv';
 import toSource from 'tosource';
 import { createFilter } from '@rollup/pluginutils';
+import stripBom from 'strip-bom';
 
 const parsers = { '.csv': csvParse, '.tsv': tsvParse };
 
@@ -18,7 +19,7 @@ export default function dsv(options = {}) {
       const ext = extname(id);
       if (!(ext in parsers)) return null;
 
-      let rows = parsers[ext](code);
+      let rows = parsers[ext](stripBom(code));
 
       if (options.processRow) {
         rows = rows.map((row) => options.processRow(row, id) || row);

packages/dsv/test/fixtures/csv-with-bom/fruit.csv

@@ -0,0 +1,4 @@
+type,count
+apples,7
+pears,4
+bananas,5

packages/dsv/test/fixtures/csv-with-bom/main.js

@@ -0,0 +1,7 @@
+import fruit from './fruit.csv';
+
+t.deepEqual(fruit, [
+  { type: 'apples', count: '7' },
+  { type: 'pears', count: '4' },
+  { type: 'bananas', count: '5' }
+]);

packages/dsv/test/fixtures/tsv-with-bom/fruit.tsv

@@ -0,0 +1,4 @@
+type	count
+apples	7
+pears	4
+bananas	5

packages/dsv/test/fixtures/tsv-with-bom/main.js

@@ -0,0 +1,7 @@
+import fruit from './fruit.tsv';
+
+t.deepEqual(fruit, [
+  { type: 'apples', count: '7' },
+  { type: 'pears', count: '4' },
+  { type: 'bananas', count: '5' }
+]);

packages/dsv/test/test.js

@@ -22,6 +22,15 @@ test('converts a csv file', async (t) => {
   return testBundle(t, bundle);
 });
 
+test('converts a csv file with bom', async (t) => {
+  const bundle = await rollup({
+    input: 'fixtures/csv-with-bom/main.js',
+    plugins: [dsv()]
+  });
+  t.plan(1);
+  return testBundle(t, bundle);
+});
+
 test('converts a tsv file', async (t) => {
   const bundle = await rollup({
     input: 'fixtures/basic-tsv/main.js',
@@ -31,6 +40,15 @@ test('converts a tsv file', async (t) => {
   return testBundle(t, bundle);
 });
 
+test('converts a tsv file with bom', async (t) => {
+  const bundle = await rollup({
+    input: 'fixtures/tsv-with-bom/main.js',
+    plugins: [dsv()]
+  });
+  t.plan(1);
+  return testBundle(t, bundle);
+});
+
 test('uses a custom processor', async (t) => {
   const parse = (value) => (isNaN(+value) ? value : +value);