From 1606d7f379307d739ffd9510d1ad01e894ed34f2 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Fri, 12 Jul 2024 09:47:38 -0500 Subject: [PATCH 01/13] Add jsonschema based type checking for sidecar entry values. --- bids-validator/src/deps/ajv.ts | 4 ++ bids-validator/src/issues/list.ts | 4 ++ bids-validator/src/schema/applyRules.ts | 27 +++++++++++++ bids-validator/src/schema/context.ts | 39 ++++++++++++++++++- bids-validator/src/types/schema.ts | 4 ++ bids-validator/src/utils/objectPathHandler.ts | 3 ++ bids-validator/src/validators/bids.ts | 4 +- 7 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 bids-validator/src/deps/ajv.ts diff --git a/bids-validator/src/deps/ajv.ts b/bids-validator/src/deps/ajv.ts new file mode 100644 index 000000000..e8ae699ac --- /dev/null +++ b/bids-validator/src/deps/ajv.ts @@ -0,0 +1,4 @@ +export {Ajv} from 'https://esm.sh/ajv@8.16.0'; +import ValidateFunction from 'https://esm.sh/ajv@8.16.0'; +import JSONSchemaType from 'https://esm.sh/ajv@8.16.0'; +export { ValidateFunction, JSONSchemaType } diff --git a/bids-validator/src/issues/list.ts b/bids-validator/src/issues/list.ts index 6310d51aa..fc8ddae1a 100644 --- a/bids-validator/src/issues/list.ts +++ b/bids-validator/src/issues/list.ts @@ -47,6 +47,10 @@ export const filenameIssues: IssueDefinitionRecord = { severity: "warning", reason: "A data files JSON sidecar is missing a key listed as recommended.", }, + JSON_SCHEMA_VALIDATION_ERROR: { + severity: "error", + reason: "Invalid JSON sidecar file. The sidecar is not formatted according the schema.", + }, TSV_ERROR: { severity: "error", reason: "generic place holder for errors from tsv files", diff --git a/bids-validator/src/schema/applyRules.ts b/bids-validator/src/schema/applyRules.ts index 5bd2ce47f..253404e46 100644 --- a/bids-validator/src/schema/applyRules.ts +++ b/bids-validator/src/schema/applyRules.ts @@ -171,6 +171,7 @@ function schemaObjectTypeCheck( if (value === "n/a") { return true; } + if ("anyOf" in schemaObject) { return schemaObject.anyOf.some((x) => schemaObjectTypeCheck(x, value, schema) @@ -179,6 +180,7 @@ function schemaObjectTypeCheck( if ("enum" in schemaObject && schemaObject.enum) { return schemaObject.enum.some((x) => x === value); } + // @ts-expect-error const format = schema.objects.formats[schemaObject.type]; const re = new RegExp(`^${format.pattern}$`); @@ -442,6 +444,31 @@ function evalJsonCheck( ]); } } + let originFileKey = '' + if (keyName in context.sidecarKeyOrigin) { + originFileKey = `${context.sidecarKeyOrigin[keyName]}.${keyName}` + } + if (context.dataset.sidecarKeyValidated.has(originFileKey)) { + return + } + if (keyName in context.sidecar) { + const validate = context.dataset.ajv.compile(schema.objects.metadata[key]) + const result = validate(context.sidecar[keyName]) + if (result === false) { + for (let error of validate.errors) { + const message = 'message' in error ? `message: ${error['message']}` : '' + context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ + { + ...context.file, + evidence: `Failed for this file.key: ${originFileKey} ${message}` + } + ]) + } + } + if (originFileKey) { + context.dataset.sidecarKeyValidated.add(originFileKey) + } + } } } diff --git a/bids-validator/src/schema/context.ts b/bids-validator/src/schema/context.ts index 094e3e277..19839717a 100644 --- a/bids-validator/src/schema/context.ts +++ b/bids-validator/src/schema/context.ts @@ -16,6 +16,11 @@ import { loadHeader } from '../files/nifti.ts' import { buildAssociations } from './associations.ts' import { ValidatorOptions } from '../setup/options.ts' import { logger } from '../utils/logger.ts' +import {Ajv, ValidateFunction, JSONSchemaType} from "../deps/ajv.ts"; +import { memoize } from "../utils/memoize.ts"; +import { + GenericSchema, +} from "../types/schema.ts"; export class BIDSContextDataset implements ContextDataset { dataset_description: Record @@ -25,13 +30,21 @@ export class BIDSContextDataset implements ContextDataset { ignored: any[] modalities: any[] subjects?: ContextDatasetSubjects + ajv: Ajv + sidecarKeyValidated: Set - constructor(options?: ValidatorOptions, description = {}) { + constructor(options?: ValidatorOptions, schema?: GenericSchema, description = {}) { this.dataset_description = description this.files = [] this.tree = {} this.ignored = [] this.modalities = [] + this.ajv = new Ajv({strictSchema: false}) + this.ajv.compile = memoize(this.ajv.compile) + this.sidecarKeyValidated = new Set() + if (schema) { + this.setCustomAjvFormats(schema) + } if (options) { this.options = options } @@ -44,6 +57,27 @@ export class BIDSContextDataset implements ContextDataset { this.dataset_description.DatasetType = 'raw' } } + + setCustomAjvFormats(schema: GenericSchema): void { + if (typeof schema['objects.formats'] !== "object") { + // logger.warning( + console.log( + `schema.objects.formats missing from schema, format validation disabled.`, + ) + return + } + const schemaFormats = schema['objects.formats'] + for (let key of Object.keys(schemaFormats)) { + if (typeof schemaFormats[key]['pattern'] !== 'string') { + // logger.warning( + console.log( + `schema.objects.formats.${key} pattern missing or invalid. Skipping this format for addition to context json validator`, + ) + continue + } + this.ajv.addFormat(key, schemaFormats[key]['pattern']) + } + } } export class BIDSContextDatasetSubjects implements ContextDatasetSubjects { @@ -78,6 +112,7 @@ export class BIDSContext implements Context { datatype: string modality: string sidecar: Record + sidecarKeyOrigin: Record json: object columns: ColumnsMap associations: ContextAssociations @@ -102,6 +137,7 @@ export class BIDSContext implements Context { this.datatype = '' this.modality = '' this.sidecar = {} + this.sidecarKeyOrigin = {} this.columns = new ColumnsMap() this.json = {} this.associations = {} as ContextAssociations @@ -163,6 +199,7 @@ export class BIDSContext implements Context { .then((text) => JSON.parse(text)) .catch((error) => {}) this.sidecar = { ...this.sidecar, ...json } + Object.keys(json).map(x => this.sidecarKeyOrigin[x] = validSidecars[0].path) } const nextDir = fileTree.directories.find((directory) => { return this.file.path.startsWith(directory.path) diff --git a/bids-validator/src/types/schema.ts b/bids-validator/src/types/schema.ts index 6862cd527..424751deb 100644 --- a/bids-validator/src/types/schema.ts +++ b/bids-validator/src/types/schema.ts @@ -81,7 +81,11 @@ export interface SchemaFields { interface SchemaType { type: string + format?: string enum?: string[] + items?: SchemaType[] + minItems?: number + maxItems?: number } interface AnyOf { diff --git a/bids-validator/src/utils/objectPathHandler.ts b/bids-validator/src/utils/objectPathHandler.ts index 91736280a..f98f59c42 100644 --- a/bids-validator/src/utils/objectPathHandler.ts +++ b/bids-validator/src/utils/objectPathHandler.ts @@ -9,6 +9,9 @@ export const hasProp = ( export const objectPathHandler = { get(target: unknown, property: string) { let res = target + if (typeof property === "symbol") { + return res + } for (const prop of property.split('.')) { if (hasProp(res, prop)) { res = res[prop] diff --git a/bids-validator/src/validators/bids.ts b/bids-validator/src/validators/bids.ts index 395784e5e..356e63d91 100644 --- a/bids-validator/src/validators/bids.ts +++ b/bids-validator/src/validators/bids.ts @@ -49,9 +49,9 @@ export async function validate( if (ddFile) { const description = await ddFile.text().then((text) => JSON.parse(text)); summary.dataProcessed = description.DatasetType === "derivative"; - dsContext = new BIDSContextDataset(options, description); + dsContext = new BIDSContextDataset(options, schema, description); } else { - dsContext = new BIDSContextDataset(options); + dsContext = new BIDSContextDataset(options, schema); issues.addNonSchemaIssue('MISSING_DATASET_DESCRIPTION', [] as IssueFile[]); } From 27a4c45ed548dabcf0c27c95fd2449d06425ca57 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Fri, 12 Jul 2024 10:20:16 -0500 Subject: [PATCH 02/13] fix some ts errors, ignore some to get tests going --- bids-validator/src/schema/applyRules.ts | 32 +++++++++++++++++-------- bids-validator/src/schema/context.ts | 11 +++++---- bids-validator/src/types/context.ts | 4 ++++ bids-validator/src/types/schema.ts | 1 + 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/bids-validator/src/schema/applyRules.ts b/bids-validator/src/schema/applyRules.ts index 253404e46..574d871a8 100644 --- a/bids-validator/src/schema/applyRules.ts +++ b/bids-validator/src/schema/applyRules.ts @@ -426,7 +426,7 @@ function evalJsonCheck( for (const [key, requirement] of Object.entries(rule.fields)) { const severity = getFieldSeverity(requirement, context); // @ts-expect-error - const keyName = schema.objects.metadata[key].name; + const keyName: string = schema.objects.metadata[key].name; if (severity && severity !== "ignore" && !(keyName in context.sidecar)) { if (requirement.issue?.code && requirement.issue?.message) { context.issues.add({ @@ -451,18 +451,30 @@ function evalJsonCheck( if (context.dataset.sidecarKeyValidated.has(originFileKey)) { return } - if (keyName in context.sidecar) { - const validate = context.dataset.ajv.compile(schema.objects.metadata[key]) + + if (keyName in context.sidecar && schema.objects.metadata && keyName in schema.objects.metadata) { + // @ts-expect-error + const validate = context.dataset.ajv.compile(schema.objects.metadata[keyName]) const result = validate(context.sidecar[keyName]) if (result === false) { - for (let error of validate.errors) { - const message = 'message' in error ? `message: ${error['message']}` : '' - context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ - { - ...context.file, - evidence: `Failed for this file.key: ${originFileKey} ${message}` - } + let messages = [] + if (!validate.errors) { + context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ + { + ...context.file, + evidence: `Failed for this file.key: ${originFileKey}` + } ]) + } else { + for (let error of validate.errors) { + const message = 'message' in error ? `message: ${error['message']}` : '' + context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ + { + ...context.file, + evidence: `Failed for this file.key: ${originFileKey} ${message}` + } + ]) + } } } if (originFileKey) { diff --git a/bids-validator/src/schema/context.ts b/bids-validator/src/schema/context.ts index 19839717a..aec6a3cb9 100644 --- a/bids-validator/src/schema/context.ts +++ b/bids-validator/src/schema/context.ts @@ -19,7 +19,7 @@ import { logger } from '../utils/logger.ts' import {Ajv, ValidateFunction, JSONSchemaType} from "../deps/ajv.ts"; import { memoize } from "../utils/memoize.ts"; import { - GenericSchema, + Schema } from "../types/schema.ts"; export class BIDSContextDataset implements ContextDataset { @@ -33,13 +33,14 @@ export class BIDSContextDataset implements ContextDataset { ajv: Ajv sidecarKeyValidated: Set - constructor(options?: ValidatorOptions, schema?: GenericSchema, description = {}) { + constructor(options?: ValidatorOptions, schema?: Schema, description = {}) { this.dataset_description = description this.files = [] this.tree = {} this.ignored = [] this.modalities = [] this.ajv = new Ajv({strictSchema: false}) + // @ts-expect-error this.ajv.compile = memoize(this.ajv.compile) this.sidecarKeyValidated = new Set() if (schema) { @@ -58,15 +59,15 @@ export class BIDSContextDataset implements ContextDataset { } } - setCustomAjvFormats(schema: GenericSchema): void { - if (typeof schema['objects.formats'] !== "object") { + setCustomAjvFormats(schema: Schema): void { + if (typeof schema.objects.formats !== "object") { // logger.warning( console.log( `schema.objects.formats missing from schema, format validation disabled.`, ) return } - const schemaFormats = schema['objects.formats'] + const schemaFormats = schema.objects.formats for (let key of Object.keys(schemaFormats)) { if (typeof schemaFormats[key]['pattern'] !== 'string') { // logger.warning( diff --git a/bids-validator/src/types/context.ts b/bids-validator/src/types/context.ts index 4b47ef0cf..cbc2e2e84 100644 --- a/bids-validator/src/types/context.ts +++ b/bids-validator/src/types/context.ts @@ -1,4 +1,5 @@ import { ValidatorOptions } from '../setup/options.ts' +import { Ajv } from "../deps/ajv.ts"; export interface ContextDatasetSubjects { sub_dirs: string[] @@ -13,6 +14,8 @@ export interface ContextDataset { modalities: any[] subjects?: ContextDatasetSubjects options?: ValidatorOptions + ajv: Ajv + sidecarKeyValidated: Set } export interface ContextSubjectSessions { ses_dirs: string[] @@ -97,6 +100,7 @@ export interface Context { extension: string modality: string sidecar: Record + sidecarKeyOrigin: Record associations: ContextAssociations columns: object json: object diff --git a/bids-validator/src/types/schema.ts b/bids-validator/src/types/schema.ts index 424751deb..b8a4f86f1 100644 --- a/bids-validator/src/types/schema.ts +++ b/bids-validator/src/types/schema.ts @@ -71,6 +71,7 @@ export interface GenericRule { format?: string required?: string index_columns?: string[] + metadata?: Record } export interface SchemaFields { From 9231f97ccbbbfc6430d086ad26a43fc746dae632 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Fri, 12 Jul 2024 10:42:52 -0500 Subject: [PATCH 03/13] add schemapath to json validation error message --- bids-validator/src/schema/applyRules.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bids-validator/src/schema/applyRules.ts b/bids-validator/src/schema/applyRules.ts index 574d871a8..6ff0eadac 100644 --- a/bids-validator/src/schema/applyRules.ts +++ b/bids-validator/src/schema/applyRules.ts @@ -457,12 +457,13 @@ function evalJsonCheck( const validate = context.dataset.ajv.compile(schema.objects.metadata[keyName]) const result = validate(context.sidecar[keyName]) if (result === false) { + const evidenceBase = `Failed for this file.key: ${originFileKey} Schema path: ${schemaPath}` let messages = [] if (!validate.errors) { context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ { ...context.file, - evidence: `Failed for this file.key: ${originFileKey}` + evidence: evidenceBase } ]) } else { @@ -471,7 +472,7 @@ function evalJsonCheck( context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ { ...context.file, - evidence: `Failed for this file.key: ${originFileKey} ${message}` + evidence: `${evidenceBase} ${message}` } ]) } From 90032c4a3897c3e4130f6e2f71c71ecaa1db33e0 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Fri, 12 Jul 2024 10:45:40 -0500 Subject: [PATCH 04/13] Update bids-validator/src/schema/applyRules.ts Co-authored-by: Chris Markiewicz --- bids-validator/src/schema/applyRules.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bids-validator/src/schema/applyRules.ts b/bids-validator/src/schema/applyRules.ts index 6ff0eadac..fdbad4757 100644 --- a/bids-validator/src/schema/applyRules.ts +++ b/bids-validator/src/schema/applyRules.ts @@ -446,7 +446,7 @@ function evalJsonCheck( } let originFileKey = '' if (keyName in context.sidecarKeyOrigin) { - originFileKey = `${context.sidecarKeyOrigin[keyName]}.${keyName}` + originFileKey = `${context.sidecarKeyOrigin[keyName]}:${keyName}` } if (context.dataset.sidecarKeyValidated.has(originFileKey)) { return From 5b00b5fa1344636177aeb90b8e86654f09e98d7b Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Mon, 15 Jul 2024 14:16:37 -0500 Subject: [PATCH 05/13] put all ajv exports in a single line with import type modifier as needed --- bids-validator/src/deps/ajv.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bids-validator/src/deps/ajv.ts b/bids-validator/src/deps/ajv.ts index e8ae699ac..fc6116de2 100644 --- a/bids-validator/src/deps/ajv.ts +++ b/bids-validator/src/deps/ajv.ts @@ -1,4 +1 @@ -export {Ajv} from 'https://esm.sh/ajv@8.16.0'; -import ValidateFunction from 'https://esm.sh/ajv@8.16.0'; -import JSONSchemaType from 'https://esm.sh/ajv@8.16.0'; -export { ValidateFunction, JSONSchemaType } +export {Ajv, type ValidateFunction, type JSONSchemaType } from 'https://esm.sh/ajv@8.16.0'; From 13e666c145fb0a7fca465f4488c7cc97e27fd196 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Mon, 15 Jul 2024 14:39:39 -0500 Subject: [PATCH 06/13] Update bids-validator/src/schema/applyRules.ts Co-authored-by: Nell Hardcastle --- bids-validator/src/schema/applyRules.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/bids-validator/src/schema/applyRules.ts b/bids-validator/src/schema/applyRules.ts index fdbad4757..d8b3c2c6c 100644 --- a/bids-validator/src/schema/applyRules.ts +++ b/bids-validator/src/schema/applyRules.ts @@ -458,7 +458,6 @@ function evalJsonCheck( const result = validate(context.sidecar[keyName]) if (result === false) { const evidenceBase = `Failed for this file.key: ${originFileKey} Schema path: ${schemaPath}` - let messages = [] if (!validate.errors) { context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ { From 52e1a74bb3ded06ae567eccbe88bf0502bb32d95 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Mon, 15 Jul 2024 14:41:13 -0500 Subject: [PATCH 07/13] add logger warning if filekey origin map not populated in context. Bail out of json validation earlier if key not in sidecar --- bids-validator/src/schema/applyRules.ts | 60 ++++++++++++++----------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/bids-validator/src/schema/applyRules.ts b/bids-validator/src/schema/applyRules.ts index fdbad4757..f2a8ecb10 100644 --- a/bids-validator/src/schema/applyRules.ts +++ b/bids-validator/src/schema/applyRules.ts @@ -444,43 +444,51 @@ function evalJsonCheck( ]); } } + + /* Regardless of if key is required/recommended/optional, we do no + * further valdiation if it is not present in sidecar. + */ + if (!(keyName in context.sidecar)) { + return + } + let originFileKey = '' if (keyName in context.sidecarKeyOrigin) { originFileKey = `${context.sidecarKeyOrigin[keyName]}:${keyName}` + } else { + logger.warning(`sidecarKeyOrigin map failed to initialize for ${context.file.path} on key ${keyName}. Validation caching not active for this key.`) } + if (context.dataset.sidecarKeyValidated.has(originFileKey)) { return } - if (keyName in context.sidecar && schema.objects.metadata && keyName in schema.objects.metadata) { - // @ts-expect-error - const validate = context.dataset.ajv.compile(schema.objects.metadata[keyName]) - const result = validate(context.sidecar[keyName]) - if (result === false) { - const evidenceBase = `Failed for this file.key: ${originFileKey} Schema path: ${schemaPath}` - let messages = [] - if (!validate.errors) { - context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ - { - ...context.file, - evidence: evidenceBase - } + // @ts-expect-error + const validate = context.dataset.ajv.compile(schema.objects.metadata[keyName]) + const result = validate(context.sidecar[keyName]) + if (result === false) { + const evidenceBase = `Failed for this file.key: ${originFileKey} Schema path: ${schemaPath}` + if (!validate.errors) { + context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ + { + ...context.file, + evidence: evidenceBase + } + ]) + } else { + for (let error of validate.errors) { + const message = 'message' in error ? `message: ${error['message']}` : '' + context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ + { + ...context.file, + evidence: `${evidenceBase} ${message}` + } ]) - } else { - for (let error of validate.errors) { - const message = 'message' in error ? `message: ${error['message']}` : '' - context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ - { - ...context.file, - evidence: `${evidenceBase} ${message}` - } - ]) - } } } - if (originFileKey) { - context.dataset.sidecarKeyValidated.add(originFileKey) - } + } + if (originFileKey) { + context.dataset.sidecarKeyValidated.add(originFileKey) } } } From 9ed3e6eba57ab268d04822643670fc5c8d003226 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 18 Jul 2024 14:32:40 -0500 Subject: [PATCH 08/13] bump bids-examples --- bids-validator/tests/data/bids-examples | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bids-validator/tests/data/bids-examples b/bids-validator/tests/data/bids-examples index 6470461ce..ea3f7f65d 160000 --- a/bids-validator/tests/data/bids-examples +++ b/bids-validator/tests/data/bids-examples @@ -1 +1 @@ -Subproject commit 6470461ce3c357080e39a11d7d7b1c4cde9f567b +Subproject commit ea3f7f65d68850630ab7c550cbe62aac9fceec47 From fd1575a96159882db899053ad84261266b4572b9 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 18 Jul 2024 16:27:37 -0500 Subject: [PATCH 09/13] bump examples --- bids-validator/tests/data/bids-examples | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bids-validator/tests/data/bids-examples b/bids-validator/tests/data/bids-examples index ea3f7f65d..401f4cfc0 160000 --- a/bids-validator/tests/data/bids-examples +++ b/bids-validator/tests/data/bids-examples @@ -1 +1 @@ -Subproject commit ea3f7f65d68850630ab7c550cbe62aac9fceec47 +Subproject commit 401f4cfc0709be0e39c32766b79894160a8a3e45 From 6b61737431e9d781fb905e205e28a9ddc12fc523 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 29 Jul 2024 18:43:10 -0400 Subject: [PATCH 10/13] FIX: Compile correct metadata definition --- bids-validator/src/schema/applyRules.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bids-validator/src/schema/applyRules.ts b/bids-validator/src/schema/applyRules.ts index 950f83c55..3417f2fa6 100644 --- a/bids-validator/src/schema/applyRules.ts +++ b/bids-validator/src/schema/applyRules.ts @@ -415,7 +415,8 @@ function evalJsonCheck( for (const [key, requirement] of Object.entries(rule.fields)) { const severity = getFieldSeverity(requirement, context) // @ts-expect-error - const keyName: string = schema.objects.metadata[key].name; + const metadataDef = schema.objects.metadata[key]; + const keyName: string = metadataDef.name; if (severity && severity !== "ignore" && !(keyName in context.sidecar)) { if (requirement.issue?.code && requirement.issue?.message) { context.issues.add({ @@ -452,8 +453,7 @@ function evalJsonCheck( return } - // @ts-expect-error - const validate = context.dataset.ajv.compile(schema.objects.metadata[keyName]) + const validate = context.dataset.ajv.compile(metadataDef) const result = validate(context.sidecar[keyName]) if (result === false) { const evidenceBase = `Failed for this file.key: ${originFileKey} Schema path: ${schemaPath}` From 8b5ec124d0f9dd77cac27245577b50927a2542f4 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 29 Jul 2024 19:04:25 -0400 Subject: [PATCH 11/13] Remove unused ts-expect-error --- bids-validator/src/schema/applyRules.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/bids-validator/src/schema/applyRules.ts b/bids-validator/src/schema/applyRules.ts index 0a3e493aa..16bc07fcc 100644 --- a/bids-validator/src/schema/applyRules.ts +++ b/bids-validator/src/schema/applyRules.ts @@ -175,7 +175,6 @@ function schemaObjectTypeCheck( return schemaObject.enum.some((x) => x === value) } - // @ts-expect-error const format = schemaObject.format // @ts-expect-error ? schema.objects.formats[schemaObject.format] From 92cbef9624663b719b7b4c22b6104669eb518f09 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 29 Jul 2024 19:08:32 -0400 Subject: [PATCH 12/13] chore(fmt): deno fmt [ignore-rev] --- bids-validator/src/deps/ajv.ts | 2 +- bids-validator/src/issues/list.ts | 4 +-- bids-validator/src/schema/applyRules.ts | 32 +++++++++---------- bids-validator/src/schema/context.ts | 24 +++++++------- bids-validator/src/types/context.ts | 2 +- bids-validator/src/utils/objectPathHandler.ts | 2 +- bids-validator/src/validators/bids.ts | 10 +++--- 7 files changed, 37 insertions(+), 39 deletions(-) diff --git a/bids-validator/src/deps/ajv.ts b/bids-validator/src/deps/ajv.ts index fc6116de2..d42c5ad69 100644 --- a/bids-validator/src/deps/ajv.ts +++ b/bids-validator/src/deps/ajv.ts @@ -1 +1 @@ -export {Ajv, type ValidateFunction, type JSONSchemaType } from 'https://esm.sh/ajv@8.16.0'; +export { Ajv, type JSONSchemaType, type ValidateFunction } from 'https://esm.sh/ajv@8.16.0' diff --git a/bids-validator/src/issues/list.ts b/bids-validator/src/issues/list.ts index eacc89001..8af740533 100644 --- a/bids-validator/src/issues/list.ts +++ b/bids-validator/src/issues/list.ts @@ -43,8 +43,8 @@ export const filenameIssues: IssueDefinitionRecord = { reason: 'A data files JSON sidecar is missing a key listed as recommended.', }, JSON_SCHEMA_VALIDATION_ERROR: { - severity: "error", - reason: "Invalid JSON sidecar file. The sidecar is not formatted according the schema.", + severity: 'error', + reason: 'Invalid JSON sidecar file. The sidecar is not formatted according the schema.', }, TSV_ERROR: { severity: 'error', diff --git a/bids-validator/src/schema/applyRules.ts b/bids-validator/src/schema/applyRules.ts index 16bc07fcc..27593b39e 100644 --- a/bids-validator/src/schema/applyRules.ts +++ b/bids-validator/src/schema/applyRules.ts @@ -166,10 +166,8 @@ function schemaObjectTypeCheck( return true } - if ("anyOf" in schemaObject) { - return schemaObject.anyOf.some((x) => - schemaObjectTypeCheck(x, value, schema) - ); + if ('anyOf' in schemaObject) { + return schemaObject.anyOf.some((x) => schemaObjectTypeCheck(x, value, schema)) } if ('enum' in schemaObject && schemaObject.enum) { return schemaObject.enum.some((x) => x === value) @@ -418,9 +416,9 @@ function evalJsonCheck( for (const [key, requirement] of Object.entries(rule.fields)) { const severity = getFieldSeverity(requirement, context) // @ts-expect-error - const metadataDef = schema.objects.metadata[key]; - const keyName: string = metadataDef.name; - if (severity && severity !== "ignore" && !(keyName in context.sidecar)) { + const metadataDef = schema.objects.metadata[key] + const keyName: string = metadataDef.name + if (severity && severity !== 'ignore' && !(keyName in context.sidecar)) { if (requirement.issue?.code && requirement.issue?.message) { context.issues.add({ key: requirement.issue.code, @@ -456,7 +454,9 @@ function evalJsonCheck( if (keyName in context.sidecarKeyOrigin) { originFileKey = `${context.sidecarKeyOrigin[keyName]}:${keyName}` } else { - logger.warning(`sidecarKeyOrigin map failed to initialize for ${context.file.path} on key ${keyName}. Validation caching not active for this key.`) + logger.warning( + `sidecarKeyOrigin map failed to initialize for ${context.file.path} on key ${keyName}. Validation caching not active for this key.`, + ) } if (context.dataset.sidecarKeyValidated.has(originFileKey)) { @@ -468,20 +468,20 @@ function evalJsonCheck( if (result === false) { const evidenceBase = `Failed for this file.key: ${originFileKey} Schema path: ${schemaPath}` if (!validate.errors) { - context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ - { - ...context.file, - evidence: evidenceBase - } + context.issues.addNonSchemaIssue('JSON_SCHEMA_VALIDATION_ERROR', [ + { + ...context.file, + evidence: evidenceBase, + }, ]) } else { for (let error of validate.errors) { const message = 'message' in error ? `message: ${error['message']}` : '' - context.issues.addNonSchemaIssue("JSON_SCHEMA_VALIDATION_ERROR", [ + context.issues.addNonSchemaIssue('JSON_SCHEMA_VALIDATION_ERROR', [ { ...context.file, - evidence: `${evidenceBase} ${message}` - } + evidence: `${evidenceBase} ${message}`, + }, ]) } } diff --git a/bids-validator/src/schema/context.ts b/bids-validator/src/schema/context.ts index 51bfdeb33..70031ce26 100644 --- a/bids-validator/src/schema/context.ts +++ b/bids-validator/src/schema/context.ts @@ -16,11 +16,9 @@ import { loadHeader } from '../files/nifti.ts' import { buildAssociations } from './associations.ts' import { ValidatorOptions } from '../setup/options.ts' import { logger } from '../utils/logger.ts' -import {Ajv, ValidateFunction, JSONSchemaType} from "../deps/ajv.ts"; -import { memoize } from "../utils/memoize.ts"; -import { - Schema -} from "../types/schema.ts"; +import { Ajv, JSONSchemaType, ValidateFunction } from '../deps/ajv.ts' +import { memoize } from '../utils/memoize.ts' +import { Schema } from '../types/schema.ts' export class BIDSContextDataset implements ContextDataset { dataset_description: Record @@ -39,7 +37,7 @@ export class BIDSContextDataset implements ContextDataset { this.tree = {} this.ignored = [] this.modalities = [] - this.ajv = new Ajv({strictSchema: false}) + this.ajv = new Ajv({ strictSchema: false }) // @ts-expect-error this.ajv.compile = memoize(this.ajv.compile) this.sidecarKeyValidated = new Set() @@ -60,12 +58,12 @@ export class BIDSContextDataset implements ContextDataset { } setCustomAjvFormats(schema: Schema): void { - if (typeof schema.objects.formats !== "object") { - // logger.warning( - console.log( - `schema.objects.formats missing from schema, format validation disabled.`, - ) - return + if (typeof schema.objects.formats !== 'object') { + // logger.warning( + console.log( + `schema.objects.formats missing from schema, format validation disabled.`, + ) + return } const schemaFormats = schema.objects.formats for (let key of Object.keys(schemaFormats)) { @@ -199,7 +197,7 @@ export class BIDSContext implements Context { .then((text) => JSON.parse(text)) .catch((error) => {}) this.sidecar = { ...this.sidecar, ...json } - Object.keys(json).map(x => this.sidecarKeyOrigin[x] = validSidecars[0].path) + Object.keys(json).map((x) => this.sidecarKeyOrigin[x] = validSidecars[0].path) } const nextDir = fileTree.directories.find((directory) => { return this.file.path.startsWith(directory.path) diff --git a/bids-validator/src/types/context.ts b/bids-validator/src/types/context.ts index cbc2e2e84..86f182d98 100644 --- a/bids-validator/src/types/context.ts +++ b/bids-validator/src/types/context.ts @@ -1,5 +1,5 @@ import { ValidatorOptions } from '../setup/options.ts' -import { Ajv } from "../deps/ajv.ts"; +import { Ajv } from '../deps/ajv.ts' export interface ContextDatasetSubjects { sub_dirs: string[] diff --git a/bids-validator/src/utils/objectPathHandler.ts b/bids-validator/src/utils/objectPathHandler.ts index f98f59c42..8abaa525e 100644 --- a/bids-validator/src/utils/objectPathHandler.ts +++ b/bids-validator/src/utils/objectPathHandler.ts @@ -9,7 +9,7 @@ export const hasProp = ( export const objectPathHandler = { get(target: unknown, property: string) { let res = target - if (typeof property === "symbol") { + if (typeof property === 'symbol') { return res } for (const prop of property.split('.')) { diff --git a/bids-validator/src/validators/bids.ts b/bids-validator/src/validators/bids.ts index e0fd862cc..ca448c99b 100644 --- a/bids-validator/src/validators/bids.ts +++ b/bids-validator/src/validators/bids.ts @@ -47,12 +47,12 @@ export async function validate( let dsContext if (ddFile) { - const description = await ddFile.text().then((text) => JSON.parse(text)); - summary.dataProcessed = description.DatasetType === "derivative"; - dsContext = new BIDSContextDataset(options, schema, description); + const description = await ddFile.text().then((text) => JSON.parse(text)) + summary.dataProcessed = description.DatasetType === 'derivative' + dsContext = new BIDSContextDataset(options, schema, description) } else { - dsContext = new BIDSContextDataset(options, schema); - issues.addNonSchemaIssue('MISSING_DATASET_DESCRIPTION', [] as IssueFile[]); + dsContext = new BIDSContextDataset(options, schema) + issues.addNonSchemaIssue('MISSING_DATASET_DESCRIPTION', [] as IssueFile[]) } let derivatives: FileTree[] = [] From 1945f1e23b719d2a2edb73ef3b1e6d8036a0cb3e Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 29 Jul 2024 19:10:52 -0400 Subject: [PATCH 13/13] git log --grep "\[ignore-rev\]" --pretty=format:"# %ad - %ae - %s%n%H" > .git-blame-ignore-revs --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 7059d6f23..dc71356bb 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,2 +1,4 @@ +# Mon Jul 29 19:08:32 2024 -0400 - effigies@gmail.com - chore(fmt): deno fmt [ignore-rev] +92cbef9624663b719b7b4c22b6104669eb518f09 # Sun Jul 28 22:01:32 2024 -0400 - effigies@gmail.com - chore(fmt): deno fmt [ignore-rev] 1bccd0a2d09f1b7ff6a26d99c7845bf59908c421 \ No newline at end of file