From 10f83d4546d8f80de334128689a79510e4bfde89 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 5 Aug 2024 09:17:02 -0400 Subject: [PATCH 1/4] feat: Add .viewed bool on BIDSFiles --- bids-validator/src/files/browser.ts | 1 + bids-validator/src/files/deno.ts | 1 + bids-validator/src/issues/datasetIssues.test.ts | 2 ++ bids-validator/src/schema/fixtures.test.ts | 5 +++++ bids-validator/src/schema/walk.ts | 1 + bids-validator/src/tests/simple-dataset.ts | 5 +++++ bids-validator/src/types/filetree.ts | 5 ++++- 7 files changed, 19 insertions(+), 1 deletion(-) diff --git a/bids-validator/src/files/browser.ts b/bids-validator/src/files/browser.ts index c983dd64a..925b1ac2a 100644 --- a/bids-validator/src/files/browser.ts +++ b/bids-validator/src/files/browser.ts @@ -11,6 +11,7 @@ export class BIDSFileBrowser implements BIDSFile { name: string path: string parent: FileTree + viewed: boolean = false constructor(file: File, ignore: FileIgnoreRules, parent?: FileTree) { this.#file = file diff --git a/bids-validator/src/files/deno.ts b/bids-validator/src/files/deno.ts index 049a3e948..096c3df96 100644 --- a/bids-validator/src/files/deno.ts +++ b/bids-validator/src/files/deno.ts @@ -26,6 +26,7 @@ export class BIDSFileDeno implements BIDSFile { parent: FileTree #fileInfo?: Deno.FileInfo #datasetAbsPath: string + viewed: boolean = false constructor(datasetPath: string, path: string, ignore: FileIgnoreRules, parent?: FileTree) { this.#datasetAbsPath = datasetPath diff --git a/bids-validator/src/issues/datasetIssues.test.ts b/bids-validator/src/issues/datasetIssues.test.ts index 25256fbc1..42e13700b 100644 --- a/bids-validator/src/issues/datasetIssues.test.ts +++ b/bids-validator/src/issues/datasetIssues.test.ts @@ -27,6 +27,7 @@ Deno.test('DatasetIssues management class', async (t) => { ignored: false, stream: testStream, parent: root, + viewed: false, } as BIDSFile, { text, @@ -40,6 +41,7 @@ Deno.test('DatasetIssues management class', async (t) => { severity: 'warning', reason: 'Readme borked', parent: root, + viewed: false, } as IssueFile, ] issues.add({ key: 'TEST_FILES_ERROR', reason: 'Test issue', files }) diff --git a/bids-validator/src/schema/fixtures.test.ts b/bids-validator/src/schema/fixtures.test.ts index 34fb9a466..f9f87cc6c 100644 --- a/bids-validator/src/schema/fixtures.test.ts +++ b/bids-validator/src/schema/fixtures.test.ts @@ -41,6 +41,7 @@ export const dataFile = { stream: new ReadableStream(), readBytes: nullReadBytes, parent: anatFileTree, + viewed: false, } anatFileTree.files = [ @@ -54,6 +55,7 @@ anatFileTree.files = [ stream: new ReadableStream(), readBytes: async (size: number) => new TextEncoder().encode(await anatJson()), parent: anatFileTree, + viewed: false, }, ] @@ -70,6 +72,7 @@ subjectFileTree.files = [ stream: new ReadableStream(), readBytes: async (size: number) => new TextEncoder().encode(await subjectJson()), parent: subjectFileTree, + viewed: false, }, ] subjectFileTree.directories = [sessionFileTree] @@ -84,6 +87,7 @@ stimuliFileTree.files = [...Array(10).keys()].map((i) => ( stream: new ReadableStream(), readBytes: nullReadBytes, parent: stimuliFileTree, + viewed: false, } )) @@ -97,6 +101,7 @@ rootFileTree.files = [ stream: new ReadableStream(), readBytes: async (size: number) => new TextEncoder().encode(await rootJson()), parent: rootFileTree, + viewed: false, }, ] rootFileTree.directories = [stimuliFileTree, subjectFileTree] diff --git a/bids-validator/src/schema/walk.ts b/bids-validator/src/schema/walk.ts index a4ca16043..5bfa4d3f9 100644 --- a/bids-validator/src/schema/walk.ts +++ b/bids-validator/src/schema/walk.ts @@ -25,6 +25,7 @@ function pseudoFile(dir: FileTree): BIDSFile { size: [...quickWalk(dir)].reduce((acc, file) => acc + file.size, 0), ignored: dir.ignored, parent: dir.parent as FileTree, + viewed: false, ...nullFile, } } diff --git a/bids-validator/src/tests/simple-dataset.ts b/bids-validator/src/tests/simple-dataset.ts index 07947c287..a8f79cf65 100644 --- a/bids-validator/src/tests/simple-dataset.ts +++ b/bids-validator/src/tests/simple-dataset.ts @@ -17,6 +17,7 @@ anatFileTree.files = [ stream: new ReadableStream(), readBytes: nullReadBytes, parent: anatFileTree, + viewed: false, }, ] subjectFileTree.files = [] @@ -31,6 +32,7 @@ rootFileTree.files = [ stream: new ReadableStream(), readBytes: nullReadBytes, parent: rootFileTree, + viewed: false, }, { text, @@ -41,6 +43,7 @@ rootFileTree.files = [ stream: new ReadableStream(), readBytes: nullReadBytes, parent: rootFileTree, + viewed: false, }, { text, @@ -51,6 +54,7 @@ rootFileTree.files = [ stream: new ReadableStream(), readBytes: nullReadBytes, parent: rootFileTree, + viewed: false, }, { text, @@ -61,6 +65,7 @@ rootFileTree.files = [ stream: new ReadableStream(), readBytes: nullReadBytes, parent: rootFileTree, + viewed: false, }, ] rootFileTree.directories = [subjectFileTree] diff --git a/bids-validator/src/types/filetree.ts b/bids-validator/src/types/filetree.ts index cecdfd731..8205c2f3c 100644 --- a/bids-validator/src/types/filetree.ts +++ b/bids-validator/src/types/filetree.ts @@ -16,7 +16,10 @@ export interface BIDSFile { text: () => Promise // Read a range of bytes readBytes: (size: number, offset?: number) => Promise + // Access the parent directory parent: FileTree + // File has been viewed + viewed: boolean } export class FileTree { @@ -43,7 +46,7 @@ export class FileTree { return false } else if (parts.length === 1) { return ( - this.files.some((x) => x.name === parts[0]) || + this.files.some((x) => (x.name === parts[0] && (x.viewed = true))) || this.directories.some((x) => x.name === parts[0]) ) } else if (parts.length > 1) { From bc700f9cdc1d790ff82d4407edf3dd9e9d22f14e Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 5 Aug 2024 10:06:25 -0400 Subject: [PATCH 2/4] feat: Implement UNUSED_STIMULUS --- bids-validator/src/issues/list.ts | 5 ++++ bids-validator/src/validators/bids.ts | 5 +++- .../src/validators/internal/unusedFile.ts | 30 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 bids-validator/src/validators/internal/unusedFile.ts diff --git a/bids-validator/src/issues/list.ts b/bids-validator/src/issues/list.ts index 0c1b6b397..84ee57fe4 100644 --- a/bids-validator/src/issues/list.ts +++ b/bids-validator/src/issues/list.ts @@ -131,6 +131,11 @@ export const bidsIssues: IssueDefinitionRecord = { severity: 'error', reason: 'Empty files not allowed.', }, + UNUSED_STIMULUS: { + severity: 'warning', + reason: + 'There are files in the /stimuli directory that are not utilized in any _events.tsv file.', + }, } const hedIssues: IssueDefinitionRecord = { diff --git a/bids-validator/src/validators/bids.ts b/bids-validator/src/validators/bids.ts index 74a4e3f2b..f48cbdcce 100644 --- a/bids-validator/src/validators/bids.ts +++ b/bids-validator/src/validators/bids.ts @@ -13,6 +13,7 @@ import { filenameIdentify } from './filenameIdentify.ts' import { filenameValidate } from './filenameValidate.ts' import { DatasetIssues } from '../issues/datasetIssues.ts' import { emptyFile } from './internal/emptyFile.ts' +import { unusedStimulus } from './internal/unusedFile.ts' import { BIDSContext, BIDSContextDataset } from '../schema/context.ts' import { parseOptions } from '../setup/options.ts' import { hedValidate } from './hed.ts' @@ -28,7 +29,9 @@ const perContextChecks: ContextCheckFunction[] = [ hedValidate, ] -const perDSChecks: DSCheckFunction[] = [] +const perDSChecks: DSCheckFunction[] = [ + unusedStimulus, +] /** * Full BIDS schema validation entrypoint diff --git a/bids-validator/src/validators/internal/unusedFile.ts b/bids-validator/src/validators/internal/unusedFile.ts new file mode 100644 index 000000000..651712802 --- /dev/null +++ b/bids-validator/src/validators/internal/unusedFile.ts @@ -0,0 +1,30 @@ +import { GenericSchema } from '../../types/schema.ts' +import { BIDSFile, FileTree } from '../../types/filetree.ts' +import { BIDSContextDataset } from '../../schema/context.ts' + +function* walkFileTree(fileTree?: FileTree): Generator { + if (!fileTree) { + return + } + for (const file of fileTree.files) { + if (!file.ignored) { + yield file + } + } + for (const dir of fileTree.directories) { + if (!dir.ignored) { + yield* walkFileTree(dir) + } + } +} + +export async function unusedStimulus( + schema: GenericSchema, + dsContext: BIDSContextDataset, +) { + const stimDir = dsContext.tree.directories.find((dir) => dir.name === 'stimuli') + const unusedStimuli = [...walkFileTree(stimDir)].filter((stimulus) => !stimulus.viewed) + if (unusedStimuli.length) { + dsContext.issues.addNonSchemaIssue('UNUSED_STIMULUS', unusedStimuli) + } +} From a94446bad6ee33d4d5de05ed6c8b319113f90642 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 5 Aug 2024 10:17:29 -0400 Subject: [PATCH 3/4] feat: Mark as viewed files found through inheritance --- bids-validator/src/files/inheritance.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bids-validator/src/files/inheritance.ts b/bids-validator/src/files/inheritance.ts index 17511395c..5690b6987 100644 --- a/bids-validator/src/files/inheritance.ts +++ b/bids-validator/src/files/inheritance.ts @@ -30,6 +30,7 @@ export function* walkBack( ) }) if (exactMatch) { + exactMatch.viewed = true yield exactMatch } else { console.warn(` @@ -39,6 +40,7 @@ ${candidates.map((file) => `* ${file.path}`).join('\n')} `) } } else if (candidates.length === 1) { + candidates[0].viewed = true yield candidates[0] } if (!inherit) break From 73c52d1e47df3bf0060e10f57106d1d001e6acf0 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 5 Aug 2024 11:37:22 -0400 Subject: [PATCH 4/4] feat: Implement SIDECAR_WITHOUT_DATAFILE --- bids-validator/src/issues/list.ts | 4 ++++ bids-validator/src/validators/bids.ts | 3 ++- .../src/validators/internal/unusedFile.ts | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/bids-validator/src/issues/list.ts b/bids-validator/src/issues/list.ts index 84ee57fe4..3cb74a965 100644 --- a/bids-validator/src/issues/list.ts +++ b/bids-validator/src/issues/list.ts @@ -136,6 +136,10 @@ export const bidsIssues: IssueDefinitionRecord = { reason: 'There are files in the /stimuli directory that are not utilized in any _events.tsv file.', }, + SIDECAR_WITHOUT_DATAFILE: { + severity: 'error', + reason: 'A json sidecar file was found without a corresponding data file', + }, } const hedIssues: IssueDefinitionRecord = { diff --git a/bids-validator/src/validators/bids.ts b/bids-validator/src/validators/bids.ts index f48cbdcce..f91548f75 100644 --- a/bids-validator/src/validators/bids.ts +++ b/bids-validator/src/validators/bids.ts @@ -13,7 +13,7 @@ import { filenameIdentify } from './filenameIdentify.ts' import { filenameValidate } from './filenameValidate.ts' import { DatasetIssues } from '../issues/datasetIssues.ts' import { emptyFile } from './internal/emptyFile.ts' -import { unusedStimulus } from './internal/unusedFile.ts' +import { sidecarWithoutDatafile, unusedStimulus } from './internal/unusedFile.ts' import { BIDSContext, BIDSContextDataset } from '../schema/context.ts' import { parseOptions } from '../setup/options.ts' import { hedValidate } from './hed.ts' @@ -31,6 +31,7 @@ const perContextChecks: ContextCheckFunction[] = [ const perDSChecks: DSCheckFunction[] = [ unusedStimulus, + sidecarWithoutDatafile, ] /** diff --git a/bids-validator/src/validators/internal/unusedFile.ts b/bids-validator/src/validators/internal/unusedFile.ts index 651712802..e0e1dddea 100644 --- a/bids-validator/src/validators/internal/unusedFile.ts +++ b/bids-validator/src/validators/internal/unusedFile.ts @@ -28,3 +28,18 @@ export async function unusedStimulus( dsContext.issues.addNonSchemaIssue('UNUSED_STIMULUS', unusedStimuli) } } + +const standalone_json = ['dataset_description.json', 'genetic_info.json'] + +export async function sidecarWithoutDatafile( + schema: GenericSchema, + dsContext: BIDSContextDataset, +) { + const unusedSidecars = [...walkFileTree(dsContext.tree)].filter( + (file) => (!file.viewed && file.name.endsWith('.json') && + !standalone_json.includes(file.name)), + ) + if (unusedSidecars.length) { + dsContext.issues.addNonSchemaIssue('SIDECAR_WITHOUT_DATAFILE', unusedSidecars) + } +}