From 33f74d1ddba8ffb5fb01f12a2a23a6351ffcbdc2 Mon Sep 17 00:00:00 2001 From: mrodrig Date: Fri, 16 Feb 2024 23:38:52 -0500 Subject: [PATCH 1/5] feat: allow for partial matching of excludeKeys against CSV keys This will allow for the filtering of keys nested underneath a specific key prefix, such as the use case of not wanting to include the keys present in an array when `expandArrayObjects` is set to `true`. This addresses the use case presented in #244. --- src/json2csv.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/json2csv.ts b/src/json2csv.ts index 96650be..e43f5e8 100755 --- a/src/json2csv.ts +++ b/src/json2csv.ts @@ -82,7 +82,13 @@ export const Json2Csv = function(options: FullJson2CsvOptions) { function filterExcludedKeys(keyPaths: string[]) { if (options.excludeKeys) { return keyPaths.filter((keyPath) => { - return !options.excludeKeys.includes(keyPath); + for (const excludedKey of options.excludeKeys) { + const regex = new RegExp(excludedKey); + if (excludedKey === keyPath || keyPath.match(regex)) { + return false; // Exclude the key + } + } + return true; // Otherwise, include the key }); } From f3189f1ebbbaebfca31081b4b1b2d403e30670d3 Mon Sep 17 00:00:00 2001 From: mrodrig Date: Fri, 16 Feb 2024 23:46:14 -0500 Subject: [PATCH 2/5] docs: add information about excludeKeys matching for #244 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bf92009..a4d11c3 100755 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Returns the CSV `string` or rejects with an `Error` if there was an issue. * `emptyFieldValue` - Any - Value that, if specified, will be substituted in for field values that are `undefined`, `null`, or an empty string. * Default: none * `excelBOM` - Boolean - Should a unicode character be prepended to allow Excel to open a UTF-8 encoded file with non-ASCII characters present. - * `excludeKeys` - Array - Specify the keys that should be excluded from the output. + * `excludeKeys` - Array - Specify the keys that should be excluded from the output. Provided keys will also be used as a RegExp to help exclude keys under a specified prefix, such as all keys of Objects in an Array when `expandArrayObjects` is `true`. * Default: `[]` * Note: When used with `unwindArrays`, arrays present at excluded key paths will not be unwound. * `expandNestedObjects` - Boolean - Should nested objects be deep-converted to CSV? From d0ff411758380c6af93ffdfe62c0519130557f42 Mon Sep 17 00:00:00 2001 From: mrodrig Date: Sat, 17 Feb 2024 00:05:21 -0500 Subject: [PATCH 3/5] test: add test cases for #244 --- test/json2csv.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/json2csv.ts b/test/json2csv.ts index 82f1265..1a1f0ff 100644 --- a/test/json2csv.ts +++ b/test/json2csv.ts @@ -478,6 +478,41 @@ export function runTests() { assert.equal(csv, updatedCsv); }); + // Test case for #244 + it('should exclude a matched key prefix from the output when unwinding arrays', () => { + const updatedCsv = csvTestData.unwind.replace(',data.options.name', '') + .replace(/,MacBook (Pro|Air) \d+/g, '') + .replace(/,(Super|Turbo)charger/g, '') + .replace('5cf7ca3616c91100018844af,Computers\n', '') + // Remove duplicate lines + .replace('5cf7ca3616c91100018844bf,Cars\n', ''); + + const csv = json2csv(jsonTestData.unwind, { + unwindArrays: true, + expandArrayObjects: true, + excludeKeys: ['data.options'] + }); + + assert.equal(csv, updatedCsv); + }); + + // Test case for #244 + it('should exclude a matched key prefix from the output when unwinding arrays', () => { + const updatedCsv = csvTestData.unwind.replace(',data.category,data.options.name', '') + .replace(/,Computers,MacBook (Pro|Air) \d+/g, '') + .replace(/,Cars,(Super|Turbo)charger/g, '') + .replace('5cf7ca3616c91100018844af\n', '') + // Remove duplicate lines + .replace('5cf7ca3616c91100018844bf\n', ''); + + const csv = json2csv(jsonTestData.unwind, { + unwindArrays: true, + excludeKeys: ['data'] + }); + + assert.equal(csv, updatedCsv); + }); + it('should use a custom value parser function when provided', () => { const updatedCsv = csvTestData.trimmedFields.split('\n'); const textRow = 'Parsed Value,Parsed Value,Parsed Value,Parsed Value,Parsed Value'; From f9905f8c8f0122b0430d2b9da381985886bc1da3 Mon Sep 17 00:00:00 2001 From: mrodrig Date: Tue, 20 Feb 2024 22:55:37 -0500 Subject: [PATCH 4/5] test: add additional test case for key appearing later in key path For #244 --- test/config/testCsvFilesList.ts | 1 + test/config/testJsonFilesList.ts | 1 + test/data/csv/excludeKeyPattern.csv | 3 +++ test/data/json/excludeKeyPattern.json | 24 ++++++++++++++++++++++++ test/json2csv.ts | 10 ++++++++++ 5 files changed, 39 insertions(+) create mode 100644 test/data/csv/excludeKeyPattern.csv create mode 100644 test/data/json/excludeKeyPattern.json diff --git a/test/config/testCsvFilesList.ts b/test/config/testCsvFilesList.ts index 634d459..d1cd61f 100644 --- a/test/config/testCsvFilesList.ts +++ b/test/config/testCsvFilesList.ts @@ -50,6 +50,7 @@ const csvFileConfig = [ {key: 'falsyValues', file: '../data/csv/falsyValues.csv'}, {key: 'nestedNotUnwoundObjects', file: '../data/csv/nestedNotUnwoundObjects.csv'}, {key: 'newlineWithWrapDelimiters', file: '../data/csv/newlineWithWrapDelimiters.csv'}, + {key: 'excludeKeyPattern', file: '../data/csv/excludeKeyPattern.csv'}, ]; function readCsvFile(filePath: string) { diff --git a/test/config/testJsonFilesList.ts b/test/config/testJsonFilesList.ts index 6c53836..e617909 100644 --- a/test/config/testJsonFilesList.ts +++ b/test/config/testJsonFilesList.ts @@ -43,4 +43,5 @@ export default { quotedFieldWithNewline: require('../data/json/quotedFieldWithNewline.json'), falsyValues: require('../data/json/falsyValues.json'), newlineWithWrapDelimiters: require('../data/json/newlineWithWrapDelimiters'), + excludeKeyPattern: require('../data/json/excludeKeyPattern'), }; diff --git a/test/data/csv/excludeKeyPattern.csv b/test/data/csv/excludeKeyPattern.csv new file mode 100644 index 0000000..a205f3c --- /dev/null +++ b/test/data/csv/excludeKeyPattern.csv @@ -0,0 +1,3 @@ +id,name.arr +1,this should appear +2,this should also appear \ No newline at end of file diff --git a/test/data/json/excludeKeyPattern.json b/test/data/json/excludeKeyPattern.json new file mode 100644 index 0000000..ca25f11 --- /dev/null +++ b/test/data/json/excludeKeyPattern.json @@ -0,0 +1,24 @@ +[ + { + "id": 1, + "arr": [ + { + "name": "foo" + } + ], + "name": { + "arr": "this should appear" + } + }, + { + "id": 2, + "arr": [ + { + "name": "bar" + } + ], + "name": { + "arr": "this should also appear" + } + } +] \ No newline at end of file diff --git a/test/json2csv.ts b/test/json2csv.ts index 1a1f0ff..27a4748 100644 --- a/test/json2csv.ts +++ b/test/json2csv.ts @@ -513,6 +513,16 @@ export function runTests() { assert.equal(csv, updatedCsv); }); + // Test case for #244 + it('should exclude a matched key prefix, but not if it is not at the start of the key path', () => { + const csv = json2csv(jsonTestData.excludeKeyPattern, { + expandArrayObjects: true, + excludeKeys: ['arr'] + }); + + assert.equal(csv, csvTestData.excludeKeyPattern); + }); + it('should use a custom value parser function when provided', () => { const updatedCsv = csvTestData.trimmedFields.split('\n'); const textRow = 'Parsed Value,Parsed Value,Parsed Value,Parsed Value,Parsed Value'; From e18d66c72f1a6241aef08884459dfd1beac43519 Mon Sep 17 00:00:00 2001 From: mrodrig Date: Tue, 20 Feb 2024 22:59:09 -0500 Subject: [PATCH 5/5] fix: resolve potential excludeKeys match to a key path other than at the start of a string In a case where `excludeKeys` is set to `["arr"]`, there was a possibility that with JSON data looking like this: ``` [ { "id": 1, "arr": [ { "name": "foo" } ], "name": { "arr": "this should appear" } }, { "id": 2, "arr": [ { "name": "bar" } ], "name": { "arr": "this should also appear" } } ] ``` could potentially result in both `arr.name` and `name.arr` being excluded. By adjusting the RegExp thats being used for the matching, this fixes a potential bug that caused `name.arr` to be mistakenly excluded. --- src/json2csv.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/json2csv.ts b/src/json2csv.ts index e43f5e8..a49d4f8 100755 --- a/src/json2csv.ts +++ b/src/json2csv.ts @@ -83,7 +83,9 @@ export const Json2Csv = function(options: FullJson2CsvOptions) { if (options.excludeKeys) { return keyPaths.filter((keyPath) => { for (const excludedKey of options.excludeKeys) { - const regex = new RegExp(excludedKey); + // Only match if the excludedKey appears at the beginning of the string so we don't accidentally match a key farther down in a key path + const regex = new RegExp(`^${excludedKey}`); + if (excludedKey === keyPath || keyPath.match(regex)) { return false; // Exclude the key }