Generate JSON schema's from spec schema to validate values in sidecars.

.git-blame-ignore-revs

@@ -1,2 +1,4 @@
+# Mon Jul 29 19:08:32 2024 -0400 - [email protected] - chore(fmt): deno fmt [ignore-rev]
+92cbef9624663b719b7b4c22b6104669eb518f09
 # Sun Jul 28 22:01:32 2024 -0400 - [email protected] - chore(fmt): deno fmt [ignore-rev]
 1bccd0a2d09f1b7ff6a26d99c7845bf59908c421

bids-validator/src/deps/ajv.ts

@@ -0,0 +1 @@
+export { Ajv, type JSONSchemaType, type ValidateFunction } from 'https://esm.sh/[email protected]'

bids-validator/src/issues/list.ts

@@ -42,6 +42,10 @@ export const filenameIssues: IssueDefinitionRecord = {
     severity: 'warning',
     reason: 'A data files JSON sidecar is missing a key listed as recommended.',
   },
+  JSON_SCHEMA_VALIDATION_ERROR: {
+    severity: 'error',
+    reason: 'Invalid JSON sidecar file. The sidecar is not formatted according the schema.',
+  },
   TSV_ERROR: {
     severity: 'error',
     reason: 'generic place holder for errors from tsv files',

bids-validator/src/schema/applyRules.ts

@@ -165,13 +165,14 @@
   if (value === 'n/a') {
     return true
   }
+
   if ('anyOf' in schemaObject) {
     return schemaObject.anyOf.some((x) => schemaObjectTypeCheck(x, value, schema))
   }
   if ('enum' in schemaObject && schemaObject.enum) {
     return schemaObject.enum.some((x) => x === value)
   }
-  // @ts-expect-error
+
   const format = schemaObject.format
     // @ts-expect-error
     ? schema.objects.formats[schemaObject.format]
@@ -415,7 +416,8 @@
   for (const [key, requirement] of Object.entries(rule.fields)) {
     const severity = getFieldSeverity(requirement, context)
     // @ts-expect-error
-    const keyName = schema.objects.metadata[key].name
+    const metadataDef = schema.objects.metadata[key]
+    const keyName: string = metadataDef.name
     if (severity && severity !== 'ignore' && !(keyName in context.sidecar)) {
       if (requirement.issue?.code && requirement.issue?.message) {
         context.issues.add({
@@ -440,6 +442,53 @@
         ])
       }
     }
+
+    /* Regardless of if key is required/recommended/optional, we do no
+     * further valdiation if it is not present in sidecar.
+     */
+    if (!(keyName in context.sidecar)) {
+      return
+    }
+
+    let originFileKey = ''
+    if (keyName in context.sidecarKeyOrigin) {
+      originFileKey = `${context.sidecarKeyOrigin[keyName]}:${keyName}`
+    } else {
+      logger.warning(
+        `sidecarKeyOrigin map failed to initialize for ${context.file.path} on key ${keyName}. Validation caching not active for this key.`,
+      )
+    }
+
+    if (context.dataset.sidecarKeyValidated.has(originFileKey)) {
+      return
+    }
+
+    const validate = context.dataset.ajv.compile(metadataDef)
+    const result = validate(context.sidecar[keyName])
+    if (result === false) {
+      const evidenceBase = `Failed for this file.key: ${originFileKey} Schema path: ${schemaPath}`
+      if (!validate.errors) {
+        context.issues.addNonSchemaIssue('JSON_SCHEMA_VALIDATION_ERROR', [
+          {
+            ...context.file,
+            evidence: evidenceBase,
+          },
+        ])
+      } else {
+        for (let error of validate.errors) {
+          const message = 'message' in error ? `message: ${error['message']}` : ''
+          context.issues.addNonSchemaIssue('JSON_SCHEMA_VALIDATION_ERROR', [
+            {
+              ...context.file,
+              evidence: `${evidenceBase} ${message}`,
+            },
+          ])
+        }
+      }
+    }
+    if (originFileKey) {
+      context.dataset.sidecarKeyValidated.add(originFileKey)
+    }
   }
 }
 

bids-validator/src/schema/context.ts

@@ -16,6 +16,9 @@
 import { buildAssociations } from './associations.ts'
 import { ValidatorOptions } from '../setup/options.ts'
 import { logger } from '../utils/logger.ts'
+import { Ajv, JSONSchemaType, ValidateFunction } from '../deps/ajv.ts'
+import { memoize } from '../utils/memoize.ts'
+import { Schema } from '../types/schema.ts'
 
 export class BIDSContextDataset implements ContextDataset {
   dataset_description: Record<string, unknown>
@@ -25,13 +28,22 @@
   ignored: any[]
   modalities: any[]
   subjects?: ContextDatasetSubjects
+  ajv: Ajv
+  sidecarKeyValidated: Set<string>
 
-  constructor(options?: ValidatorOptions, description = {}) {
+  constructor(options?: ValidatorOptions, schema?: Schema, description = {}) {
     this.dataset_description = description
     this.files = []
     this.tree = {}
     this.ignored = []
     this.modalities = []
+    this.ajv = new Ajv({ strictSchema: false })
+    // @ts-expect-error
+    this.ajv.compile = memoize(this.ajv.compile)
+    this.sidecarKeyValidated = new Set<string>()
+    if (schema) {
+      this.setCustomAjvFormats(schema)
+    }
     if (options) {
       this.options = options
     }
@@ -44,6 +56,27 @@
       this.dataset_description.DatasetType = 'raw'
     }
   }
+
+  setCustomAjvFormats(schema: Schema): void {
+    if (typeof schema.objects.formats !== 'object') {
+      // logger.warning(
+      console.log(
+        `schema.objects.formats missing from schema, format validation disabled.`,
+      )
+      return
+    }
+    const schemaFormats = schema.objects.formats
+    for (let key of Object.keys(schemaFormats)) {
+      if (typeof schemaFormats[key]['pattern'] !== 'string') {
+        // logger.warning(
+        console.log(
+          `schema.objects.formats.${key} pattern missing or invalid. Skipping this format for addition to context json validator`,
+        )
+        continue
+      }
+      this.ajv.addFormat(key, schemaFormats[key]['pattern'])
+    }
+  }
 }
 
 export class BIDSContextDatasetSubjects implements ContextDatasetSubjects {
@@ -78,6 +111,7 @@
   datatype: string
   modality: string
   sidecar: Record<string, any>
+  sidecarKeyOrigin: Record<string, string>
   json: object
   columns: ColumnsMap
   associations: ContextAssociations
@@ -102,6 +136,7 @@
     this.datatype = ''
     this.modality = ''
     this.sidecar = {}
+    this.sidecarKeyOrigin = {}
     this.columns = new ColumnsMap()
     this.json = {}
     this.associations = {} as ContextAssociations
@@ -166,6 +201,7 @@
         .then((text) => JSON.parse(text))
         .catch((error) => {})
       this.sidecar = { ...this.sidecar, ...json }
+      Object.keys(json).map((x) => this.sidecarKeyOrigin[x] = validSidecars[0].path)
     }
     const nextDir = fileTree.directories.find((directory) => {
       return this.file.path.startsWith(`${directory.path}/`)

bids-validator/src/types/context.ts

@@ -1,4 +1,5 @@
 import { ValidatorOptions } from '../setup/options.ts'
+import { Ajv } from '../deps/ajv.ts'
 
 export interface ContextDatasetSubjects {
   sub_dirs: string[]
@@ -13,6 +14,8 @@ export interface ContextDataset {
   modalities: any[]
   subjects?: ContextDatasetSubjects
   options?: ValidatorOptions
+  ajv: Ajv
+  sidecarKeyValidated: Set<string>
 }
 export interface ContextSubjectSessions {
   ses_dirs: string[]
@@ -98,6 +101,7 @@ export interface Context {
   extension: string
   modality: string
   sidecar: Record<string, any>
+  sidecarKeyOrigin: Record<string, string>
   associations: ContextAssociations
   columns: object
   json: object

bids-validator/src/types/schema.ts

@@ -71,6 +71,7 @@ export interface GenericRule {
   format?: string
   required?: string
   index_columns?: string[]
+  metadata?: Record<string, string>
 }
 
 export interface SchemaFields {
@@ -81,7 +82,11 @@ export interface SchemaFields {
 
 interface SchemaType {
   type: string
+  format?: string
   enum?: string[]
+  items?: SchemaType[]
+  minItems?: number
+  maxItems?: number
 }
 
 interface AnyOf {

bids-validator/src/utils/objectPathHandler.ts

@@ -9,6 +9,9 @@
 export const objectPathHandler = {
   get(target: unknown, property: string) {
     let res = target
+    if (typeof property === 'symbol') {
+      return res
+    }
     for (const prop of property.split('.')) {
       if (hasProp(res, prop)) {
         res = res[prop]

bids-validator/src/validators/bids.ts

@@ -49,9 +49,9 @@ export async function validate(
   if (ddFile) {
     const description = await ddFile.text().then((text) => JSON.parse(text))
     summary.dataProcessed = description.DatasetType === 'derivative'
-    dsContext = new BIDSContextDataset(options, description)
+    dsContext = new BIDSContextDataset(options, schema, description)
   } else {
-    dsContext = new BIDSContextDataset(options)
+    dsContext = new BIDSContextDataset(options, schema)
     issues.addNonSchemaIssue('MISSING_DATASET_DESCRIPTION', [] as IssueFile[])
   }