Pre-compile json schemas into AJV validators (decentralized-identity#148

)

* address warnings generated when validating `public-jwk` json schema

* janky attempt to use precompiled json schema validators

* move bundling scripts & config to `build` dir

* pre-compile json schemas into ajv validators

* add comment describing `compile-validators` build script

* gitignore generated dir

.eslintignore

@@ -1,3 +1,4 @@
-node_modules
 dist
+generated
+node_modules
 try.ts

.gitignore

@@ -16,4 +16,6 @@ INDEX
 TEST-INDEX
 # folders used by code coverage
 .nyc_output/
-coverage
+coverage
+
+generated

README.md

@@ -2,7 +2,7 @@
 
 Code Coverage
 
-![Statements](https://img.shields.io/badge/statements-50.97%25-red.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-71.42%25-red.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-31.46%25-red.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-50.97%25-red.svg?style=flat)
+![Statements](https://img.shields.io/badge/statements-91.1%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-88.73%25-yellow.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-87.41%25-yellow.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-91.1%25-brightgreen.svg?style=flat)
 
 ## Introduction
 

build/compile-validators.js

@@ -0,0 +1,67 @@
+/**
+ * Pre-compiles Ajv validators from json schemas
+ * Ajv supports generating standalone validation functions from JSON Schemas at compile/build time.
+ * These functions can then be used during runtime to do validation without initializing Ajv.
+ * It is useful for several reasons:
+ * - to avoid dynamic code evaluation with Function constructor (used for schema compilation) -
+ *   when it is prohibited by the browser page [Content Security Policy](https://ajv.js.org/security.html#content-security-policy).
+ * - to reduce the browser bundle size - Ajv is not included in the bundle
+ * - to reduce the start-up time - the validation and compilation of schemas will happen during build time.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import url from 'node:url';
+
+import Ajv from 'ajv';
+import standaloneCode from 'ajv/dist/standalone';
+import mkdirp from 'mkdirp';
+
+import CollectionsQuery from '../json-schemas/collections/collections-query.json' assert { type: 'json' };
+import CollectionsWrite from '../json-schemas/collections/collections-write.json' assert { type: 'json' };
+import Definitions from '../json-schemas/definitions.json' assert { type: 'json' };
+import GeneralJwk from '../json-schemas/jwk/general-jwk.json' assert { type: 'json' };
+import GeneralJws from '../json-schemas/general-jws.json' assert { type: 'json' };
+import HooksWrite from '../json-schemas/hooks/hooks-write.json' assert { type: 'json' };
+import JwkVerificationMethod from '../json-schemas/jwk-verification-method.json' assert {type: 'json'};
+import PermissionsDefinitions from '../json-schemas/permissions/definitions.json' assert { type: 'json' };
+import PermissionsRequest from '../json-schemas/permissions/permissions-request.json' assert { type: 'json' };
+import PermissionsGrant from '../json-schemas/permissions/permissions-grant.json' assert { type: 'json' };
+import ProtocolDefinition from '../json-schemas/protocol-definition.json' assert { type: 'json' };
+import ProtocolRuleSet from '../json-schemas/protocol-rule-set.json' assert { type: 'json' };
+import ProtocolsConfigure from '../json-schemas/protocols/protocols-configure.json' assert { type: 'json' };
+import ProtocolsQuery from '../json-schemas/protocols/protocols-query.json' assert { type: 'json' };
+import PublicJwk from '../json-schemas/jwk/public-jwk.json' assert { type: 'json' };
+import Request from '../json-schemas/request.json' assert { type: 'json' };
+
+const schemas = {
+  CollectionsQuery,
+  CollectionsWrite,
+  Definitions,
+  GeneralJwk,
+  GeneralJws,
+  HooksWrite,
+  JwkVerificationMethod,
+  PermissionsDefinitions,
+  PermissionsGrant,
+  PermissionsRequest,
+  ProtocolDefinition,
+  ProtocolRuleSet,
+  ProtocolsConfigure,
+  ProtocolsQuery,
+  PublicJwk,
+  Request
+};
+
+const ajv = new Ajv({ code: { source: true, esm: true } });
+
+for (const schemaName in schemas) {
+  ajv.addSchema(schemas[schemaName], schemaName);
+}
+
+const moduleCode = standaloneCode(ajv);
+
+const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
+
+await mkdirp(path.join(__dirname, '../generated'));
+fs.writeFileSync(path.join(__dirname, '../generated/precompiled-validators.js'), moduleCode);

src/validation/json-schemas/jwk/public-jwk.json → json-schemas/jwk/public-jwk.json

@@ -5,6 +5,7 @@
   "not": {
     "anyOf": [
       {
+        "type": "object",
         "properties": {
           "kty": {
             "const": "EC"
@@ -13,6 +14,7 @@
         "anyOf": [{ "required": ["d"] }]
       },
       {
+        "type": "object",
         "properties": {
           "kty": {
             "const": "OKP"
@@ -21,6 +23,7 @@
         "anyOf": [{ "required": ["d"] }]
       },
       {
+        "type": "object",
         "properties": {
           "kty": {
             "const": "RSA"

karma.conf.cjs

@@ -1,7 +1,7 @@
 // Karma is what we're using to run our tests in browser environments
 // Karma does not support .mjs
 
-const esbuildBrowserConfig = require('./esbuild-browser-config.cjs');
+const esbuildBrowserConfig = require('./build/esbuild-browser-config.cjs');
 
 module.exports = function(config) {
   config.set({

karma.conf.debug.cjs

@@ -1,7 +1,8 @@
 // Karma is what we're using to run our tests in browser environments
 // Karma does not support .mjs
 
-const { NodeGlobalsPolyfillPlugin } = require('@esbuild-plugins/node-globals-polyfill');
+const esbuildBrowserConfig = require('./build/esbuild-browser-config.cjs');
+
 module.exports = function(config) {
   config.set({
     plugins: [
@@ -26,17 +27,7 @@ module.exports = function(config) {
       'tests/**/*.ts': ['esbuild']
     },
 
-    esbuild: {
-      mainFields : ['browser', 'module', 'main'],
-      target     : ['chrome101'],
-      plugins    : [NodeGlobalsPolyfillPlugin({
-        process: true
-      })],
-      sourcemap : true,
-      define    : {
-        'global': 'window'
-      },
-    },
+    esbuild: esbuildBrowserConfig,
 
     // list of files / patterns to exclude
     exclude: [],

package.json

@@ -21,17 +21,18 @@
   ],
   "scripts": {
     "build:esm": "tsc",
-    "build": "npm-run-all -l clean -p build:esm bundle",
-    "clean": "rimraf dist",
-    "bundle": "node create-browser-bundle.cjs",
+    "build": "npm-run-all -l clean compile-validators -p build:esm bundle",
+    "bundle": "node ./build/create-browser-bundle.cjs",
+    "clean": "rimraf dist && rimraf generated/*",
+    "compile-validators": "node --es-module-specifier-resolution=node ./build/compile-validators.js",
     "lint": "eslint . --ext .ts --max-warnings 0",
     "lint:fix": "eslint . --ext .ts --fix",
-    "test:node": "tsc && c8 mocha --es-module-specifier-resolution=node \"dist/esm/tests/**/*.spec.js\" && npm run make-coverage-badges",
-    "test:node:ci": "tsc && c8 mocha --es-module-specifier-resolution=node \"dist/esm/tests/**/*.spec.js\"",
-    "test:browser": "cross-env karma start karma.conf.cjs",
-    "test:browser-debug": "cross-env karma start karma.conf.debug.cjs",
     "make-coverage-badges": "istanbul-badges-readme",
-    "make-coverage-badges:ci": "npm run make-coverage-badges -- --ci"
+    "make-coverage-badges:ci": "npm run make-coverage-badges -- --ci",
+    "test:node": "npm run compile-validators && tsc && c8 mocha --es-module-specifier-resolution=node \"dist/esm/tests/**/*.spec.js\" && npm run make-coverage-badges",
+    "test:node:ci": "npm run compile-validators && tsc && c8 mocha --es-module-specifier-resolution=node \"dist/esm/tests/**/*.spec.js\"",
+    "test:browser": "npm run compile-validators && cross-env karma start karma.conf.cjs",
+    "test:browser-debug": "npm run compile-validators && cross-env karma start karma.conf.debug.cjs"
   },
   "dependencies": {
     "@ipld/dag-cbor": "7.0.1",
@@ -83,6 +84,7 @@
     "karma-mocha": "2.0.1",
     "karma-mocha-reporter": "2.2.5",
     "mocha": "10.1.0",
+    "mkdirp": "1.0.4",
     "mockdate": "3.0.5",
     "ms": "2.1.3",
     "npm-run-all": "4.1.5",

src/core/message.ts

@@ -6,7 +6,7 @@ import { CollectionsWriteMessage } from '../interfaces/collections/types';
 import { compareCids, generateCid } from '../utils/cid';
 import { GeneralJws } from '../jose/jws/general/types';
 import { GeneralJwsSigner, GeneralJwsVerifier } from '../jose/jws/general';
-import { validate } from '../validation/validator';
+import { validate } from '../validator';
 
 export enum DwnMethodName {
   CollectionsWrite = 'CollectionsWrite',

src/core/request.ts

@@ -1,5 +1,5 @@
 import { RequestSchema } from './types';
-import { validate } from '../validation/validator';
+import { validate } from '../validator';
 
 export class Request {
   /**

src/dwn.ts

@@ -3,7 +3,6 @@ import type { DidMethodResolver } from './did/did-resolver';
 import type { Interface, MethodHandler } from './interfaces/types';
 import type { MessageStore } from './store/message-store';
 import * as encoder from '../src/utils/encoder';
-import { addSchema } from './validation/validator';
 import { CollectionsInterface, PermissionsInterface, ProtocolsInterface } from './interfaces';
 import { DidResolver } from './did/did-resolver';
 import { MessageReply, Request, Response } from './core';
@@ -29,7 +28,7 @@ export class Dwn {
     config.messageStore ??= new MessageStoreLevel();
     config.interfaces ??= [];
 
-    for (const { methodHandlers, schemas } of config.interfaces) {
+    for (const { methodHandlers } of config.interfaces) {
 
       for (const messageType in methodHandlers) {
         if (Dwn.methodHandlers[messageType]) {
@@ -38,10 +37,6 @@ export class Dwn {
           Dwn.methodHandlers[messageType] = methodHandlers[messageType];
         }
       }
-
-      for (const schemaName in schemas) {
-        addSchema(schemaName, schemas[schemaName]);
-      }
     }
 
     const dwn = new Dwn(config);

src/jose/jws/general/verifier.ts

@@ -6,7 +6,7 @@ import * as encoder from '../../../utils/encoder';
 import { DidResolver } from '../../../did/did-resolver';
 import { MemoryCache } from '../../../utils/memory-cache';
 import { signers as verifiers } from '../../algorithms';
-import { validate } from '../../../validation/validator';
+import { validate } from '../../../validator';
 
 type VerificationResult = {
   /** DIDs of all signers */

src/validation/validator.ts → src/validator.ts

@@ -1,23 +1,13 @@
-import Ajv from 'ajv';
-import { schemas } from './json-schemas';
-
-const validator = new Ajv();
-
-for (const schemaName in schemas) {
-  addSchema(schemaName, schemas[schemaName]);
-}
-
-export function addSchema(schemaName: string, schema): void {
-  validator.addSchema(schema, schemaName);
-}
+import * as precompiledValidators from '../generated/precompiled-validators';
 
 /**
  * Validates the given payload using JSON schema keyed by the given schema name. Throws if the given payload fails validation.
  * @param schemaName the schema name use to look up the JSON schema to be used for schema validation
  * @param payload javascript object to be validated
  */
 export function validate(schemaName: string, payload: any): void {
-  const validateFn = validator.getSchema(schemaName);
+  // const validateFn = validator.getSchema(schemaName);
+  const validateFn = precompiledValidators[schemaName];
 
   if (!validateFn) {
     throw new Error(`schema for ${schemaName} not found.`);