Skip to content

Commit

Permalink
feat(csv-parse): new comment_no_infix option (fix #325)
Browse files Browse the repository at this point in the history
  • Loading branch information
@wdavidw
wdavidw committed Aug 25, 2023
1 parent 87fe919 commit caca5c3
Show file tree
Hide file tree
Showing 15 changed files with 166 additions and 19 deletions.
14 changes: 12 additions & 2 deletions packages/csv-parse/dist/cjs/index.cjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,16 @@ const normalize_options = function(opts){
], options);
}
}
// Normalize option `comment_no_infix`
if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
options.comment_no_infix = false;
}else if(options.comment_no_infix !== true){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment_no_infix:',
'value must be a boolean,',
`got ${JSON.stringify(options.comment_no_infix)}`
], options);
}
// Normalize option `delimiter`
const delimiter_json = JSON.stringify(options.delimiter);
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
Expand Down Expand Up @@ -639,7 +649,7 @@ const transform = function(original_options = {}) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter} = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
Expand Down Expand Up @@ -838,7 +848,7 @@ const transform = function(original_options = {}) {
continue;
}
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr);
if(commentCount !== 0){
if(commentCount !== 0 && (comment_no_infix === false || this.state.field.length === 0)){
this.state.commenting = true;
continue;
}
Expand Down
6 changes: 6 additions & 0 deletions packages/csv-parse/dist/cjs/index.d.cts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,12 @@ export interface Options {
* Treat all the characters after this one as a comment, default to '' (disabled).
*/
comment?: string;
/**
* Restrict the definition of comments to a full line. Comment characters
* defined in the middle of the line are not interpreted as such. The
* option require the activation of comments.
*/
comment_no_infix?: boolean;
/**
* Set the field delimiter. One character only, defaults to comma.
*/
Expand Down
14 changes: 12 additions & 2 deletions packages/csv-parse/dist/cjs/sync.cjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,16 @@ const normalize_options = function(opts){
], options);
}
}
// Normalize option `comment_no_infix`
if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
options.comment_no_infix = false;
}else if(options.comment_no_infix !== true){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment_no_infix:',
'value must be a boolean,',
`got ${JSON.stringify(options.comment_no_infix)}`
], options);
}
// Normalize option `delimiter`
const delimiter_json = JSON.stringify(options.delimiter);
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
Expand Down Expand Up @@ -637,7 +647,7 @@ const transform = function(original_options = {}) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter} = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
Expand Down Expand Up @@ -836,7 +846,7 @@ const transform = function(original_options = {}) {
continue;
}
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr);
if(commentCount !== 0){
if(commentCount !== 0 && (comment_no_infix === false || this.state.field.length === 0)){
this.state.commenting = true;
continue;
}
Expand Down
6 changes: 6 additions & 0 deletions packages/csv-parse/dist/esm/index.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,12 @@ export interface Options {
* Treat all the characters after this one as a comment, default to '' (disabled).
*/
comment?: string;
/**
* Restrict the definition of comments to a full line. Comment characters
* defined in the middle of the line are not interpreted as such. The
* option require the activation of comments.
*/
comment_no_infix?: boolean;
/**
* Set the field delimiter. One character only, defaults to comma.
*/
Expand Down
14 changes: 12 additions & 2 deletions packages/csv-parse/dist/esm/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5400,6 +5400,16 @@ const normalize_options = function(opts){
], options);
}
}
// Normalize option `comment_no_infix`
if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
options.comment_no_infix = false;
}else if(options.comment_no_infix !== true){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment_no_infix:',
'value must be a boolean,',
`got ${JSON.stringify(options.comment_no_infix)}`
], options);
}
// Normalize option `delimiter`
const delimiter_json = JSON.stringify(options.delimiter);
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
Expand Down Expand Up @@ -5761,7 +5771,7 @@ const transform = function(original_options = {}) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter} = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
Expand Down Expand Up @@ -5960,7 +5970,7 @@ const transform = function(original_options = {}) {
continue;
}
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr);
if(commentCount !== 0){
if(commentCount !== 0 && (comment_no_infix === false || this.state.field.length === 0)){
this.state.commenting = true;
continue;
}
Expand Down
14 changes: 12 additions & 2 deletions packages/csv-parse/dist/esm/sync.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2246,6 +2246,16 @@ const normalize_options = function(opts){
], options);
}
}
// Normalize option `comment_no_infix`
if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
options.comment_no_infix = false;
}else if(options.comment_no_infix !== true){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment_no_infix:',
'value must be a boolean,',
`got ${JSON.stringify(options.comment_no_infix)}`
], options);
}
// Normalize option `delimiter`
const delimiter_json = JSON.stringify(options.delimiter);
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
Expand Down Expand Up @@ -2607,7 +2617,7 @@ const transform = function(original_options = {}) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter} = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
Expand Down Expand Up @@ -2806,7 +2816,7 @@ const transform = function(original_options = {}) {
continue;
}
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr);
if(commentCount !== 0){
if(commentCount !== 0 && (comment_no_infix === false || this.state.field.length === 0)){
this.state.commenting = true;
continue;
}
Expand Down
14 changes: 12 additions & 2 deletions packages/csv-parse/dist/iife/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5403,6 +5403,16 @@ var csv_parse = (function (exports) {
], options);
}
}
// Normalize option `comment_no_infix`
if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
options.comment_no_infix = false;
}else if(options.comment_no_infix !== true){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment_no_infix:',
'value must be a boolean,',
`got ${JSON.stringify(options.comment_no_infix)}`
], options);
}
// Normalize option `delimiter`
const delimiter_json = JSON.stringify(options.delimiter);
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
Expand Down Expand Up @@ -5764,7 +5774,7 @@ var csv_parse = (function (exports) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter} = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
Expand Down Expand Up @@ -5963,7 +5973,7 @@ var csv_parse = (function (exports) {
continue;
}
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr);
if(commentCount !== 0){
if(commentCount !== 0 && (comment_no_infix === false || this.state.field.length === 0)){
this.state.commenting = true;
continue;
}
Expand Down
14 changes: 12 additions & 2 deletions packages/csv-parse/dist/iife/sync.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2249,6 +2249,16 @@ var csv_parse_sync = (function (exports) {
], options);
}
}
// Normalize option `comment_no_infix`
if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
options.comment_no_infix = false;
}else if(options.comment_no_infix !== true){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment_no_infix:',
'value must be a boolean,',
`got ${JSON.stringify(options.comment_no_infix)}`
], options);
}
// Normalize option `delimiter`
const delimiter_json = JSON.stringify(options.delimiter);
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
Expand Down Expand Up @@ -2610,7 +2620,7 @@ var csv_parse_sync = (function (exports) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter} = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
Expand Down Expand Up @@ -2809,7 +2819,7 @@ var csv_parse_sync = (function (exports) {
continue;
}
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr);
if(commentCount !== 0){
if(commentCount !== 0 && (comment_no_infix === false || this.state.field.length === 0)){
this.state.commenting = true;
continue;
}
Expand Down
14 changes: 12 additions & 2 deletions packages/csv-parse/dist/umd/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5406,6 +5406,16 @@
], options);
}
}
// Normalize option `comment_no_infix`
if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
options.comment_no_infix = false;
}else if(options.comment_no_infix !== true){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment_no_infix:',
'value must be a boolean,',
`got ${JSON.stringify(options.comment_no_infix)}`
], options);
}
// Normalize option `delimiter`
const delimiter_json = JSON.stringify(options.delimiter);
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
Expand Down Expand Up @@ -5767,7 +5777,7 @@
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter} = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
Expand Down Expand Up @@ -5966,7 +5976,7 @@
continue;
}
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr);
if(commentCount !== 0){
if(commentCount !== 0 && (comment_no_infix === false || this.state.field.length === 0)){
this.state.commenting = true;
continue;
}
Expand Down
14 changes: 12 additions & 2 deletions packages/csv-parse/dist/umd/sync.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2252,6 +2252,16 @@
], options);
}
}
// Normalize option `comment_no_infix`
if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
options.comment_no_infix = false;
}else if(options.comment_no_infix !== true){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment_no_infix:',
'value must be a boolean,',
`got ${JSON.stringify(options.comment_no_infix)}`
], options);
}
// Normalize option `delimiter`
const delimiter_json = JSON.stringify(options.delimiter);
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
Expand Down Expand Up @@ -2613,7 +2623,7 @@
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter} = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
Expand Down Expand Up @@ -2812,7 +2822,7 @@
continue;
}
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr);
if(commentCount !== 0){
if(commentCount !== 0 && (comment_no_infix === false || this.state.field.length === 0)){
this.state.commenting = true;
continue;
}
Expand Down
4 changes: 2 additions & 2 deletions packages/csv-parse/lib/api/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ const transform = function(original_options = {}) {
},
// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter} = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
let buf;
Expand Down Expand Up @@ -260,7 +260,7 @@ const transform = function(original_options = {}) {
continue;
}
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr);
if(commentCount !== 0){
if(commentCount !== 0 && (comment_no_infix === false || this.state.field.length === 0)){
this.state.commenting = true;
continue;
}
Expand Down
10 changes: 10 additions & 0 deletions packages/csv-parse/lib/api/normalize_options.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,16 @@ const normalize_options = function(opts){
], options);
}
}
// Normalize option `comment_no_infix`
if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
options.comment_no_infix = false;
}else if(options.comment_no_infix !== true){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment_no_infix:',
'value must be a boolean,',
`got ${JSON.stringify(options.comment_no_infix)}`
], options);
}
// Normalize option `delimiter`
const delimiter_json = JSON.stringify(options.delimiter);
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
Expand Down
6 changes: 6 additions & 0 deletions packages/csv-parse/lib/index.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,12 @@ export interface Options {
* Treat all the characters after this one as a comment, default to '' (disabled).
*/
comment?: string;
/**
* Restrict the definition of comments to a full line. Comment characters
* defined in the middle of the line are not interpreted as such. The
* option require the activation of comments.
*/
comment_no_infix?: boolean;
/**
* Set the field delimiter. One character only, defaults to comma.
*/
Expand Down
3 changes: 2 additions & 1 deletion packages/csv-parse/test/api.types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ describe('API Types', () => {
const options: Options = parser.options
const keys: string[] = Object.keys(options)
keys.sort().should.eql([
'bom', 'cast', 'cast_date', 'cast_first_line_to_header', 'cast_function', 'columns', 'comment', 'delimiter',
'bom', 'cast', 'cast_date', 'cast_first_line_to_header',
'cast_function', 'columns', 'comment', 'comment_no_infix', 'delimiter',
'encoding', 'escape', 'from', 'from_line', 'group_columns_by_name',
'ignore_last_delimiters', 'info', 'ltrim', 'max_record_size', 'objname',
'on_record', 'on_skip', 'quote', 'raw', 'record_delimiter',
Expand Down
Loading

0 comments on commit caca5c3

Please sign in to comment.