From aa669eaa887092ddf4cfd2c85d7e1061dae3503b Mon Sep 17 00:00:00 2001 From: Alejandro Villar Date: Tue, 19 Dec 2023 09:56:40 +0100 Subject: [PATCH] Use nodejs to further validate JSON-LD contexts --- .gitignore | 1 + Dockerfile | 6 +- ogc/bblocks/postprocess.py | 20 +- ogc/bblocks/util.py | 6 +- ogc/bblocks/validation/validate-jsonld.js | 23 +++ package-lock.json | 233 ++++++++++++++++++++++ package.json | 5 + 7 files changed, 285 insertions(+), 9 deletions(-) create mode 100644 ogc/bblocks/validation/validate-jsonld.js create mode 100644 package-lock.json create mode 100644 package.json diff --git a/.gitignore b/.gitignore index 083d732..eb49c8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__ .idea +node_modules/ diff --git a/Dockerfile b/Dockerfile index 1521c71..2283c2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,15 +4,17 @@ WORKDIR /src COPY requirements.txt / RUN apk update && \ - apk add git rsync && \ + apk add git rsync nodejs npm && \ python -m venv /venv && \ /venv/bin/python -m pip install --upgrade pip && \ - git config --global --add safe.directory '*' + git config --global --add safe.directory '*' && \ + npm install jsonld RUN /venv/bin/python -m pip install -r /requirements.txt ENV PYTHONPATH /src/ ENV PYTHONUNBUFFERED 1 +ENV NODE_PATH=/src/node_modules COPY ogc/ /src/ogc/ diff --git a/ogc/bblocks/postprocess.py b/ogc/bblocks/postprocess.py index 2590d9c..ccc5985 100644 --- a/ogc/bblocks/postprocess.py +++ b/ogc/bblocks/postprocess.py @@ -35,7 +35,7 @@ def postprocess(registered_items_path: str | Path = 'registereditems', github_base_url: str | None = None, imported_registers: list[str] | None = None, bb_filter: str | None = None, - steps: list[str] | None = None) -> list[BuildingBlock]: + steps: list[str] | None = None) -> list[dict]: cwd = Path().resolve() @@ -124,7 +124,7 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool: if base_url: bblock.metadata['sourceFiles'] = f"{base_url}{rel_files_path}/" else: - bblock.metadata['sourceFiles'] = f"./{os.path.relpath(rel_files_path, output_file_root)}/" + bblock.metadata['sourceFiles'] = f"./{os.path.relpath(str(rel_files_path), str(output_file_root))}/" if not light: if not steps or 'tests' in steps: @@ -170,9 +170,6 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool: doc_generator.generate_doc(bblock) return True - if not isinstance(registered_items_path, Path): - registered_items_path = Path(registered_items_path) - filter_id = None if bb_filter: filter_id = False @@ -243,6 +240,19 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool: try: written_context = write_jsonld_context(building_block.annotated_schema) if written_context: + try: + nodejsrun = subprocess.run([ + 'node', + str(Path(__file__).parent.joinpath('validation/validate-jsonld.js')), + str(written_context), + ], capture_output=True) + if nodejsrun.returncode == 26: # validation error + raise ValueError(nodejsrun.stdout.decode()) + elif nodejsrun.returncode == 0: + written_context = f"{written_context} (validated)" + except FileNotFoundError: + # node not installed + pass print(f" - {written_context}", file=sys.stderr) except Exception as e: if fail_on_error: diff --git a/ogc/bblocks/util.py b/ogc/bblocks/util.py index 6e89ce4..58df600 100644 --- a/ogc/bblocks/util.py +++ b/ogc/bblocks/util.py @@ -467,11 +467,13 @@ def ref_updater(ref): return result -def write_jsonld_context(annotated_schema: Path) -> Path | None: +def write_jsonld_context(annotated_schema: Path | str) -> Path | None: + if not isinstance(annotated_schema, Path): + annotated_schema = Path(annotated_schema) ctx_builder = ContextBuilder(annotated_schema) if not ctx_builder.context.get('@context'): return None - context_fn = annotated_schema.parent / 'context.jsonld' + context_fn = annotated_schema.resolve().parent / 'context.jsonld' with open(context_fn, 'w') as f: json.dump(ctx_builder.context, f, indent=2) with open(context_fn.parent / '_visited_properties.tsv', 'w', newline='') as f: diff --git a/ogc/bblocks/validation/validate-jsonld.js b/ogc/bblocks/validation/validate-jsonld.js new file mode 100644 index 0000000..4689da1 --- /dev/null +++ b/ogc/bblocks/validation/validate-jsonld.js @@ -0,0 +1,23 @@ +const jsonld = require('jsonld'); +const util = require('util'); +const fs = require('fs'); + +if (process.argv.length !== 3) { + console.error('File name to validate required'); + process.exit(25); +} + +const filename = process.argv[2]; +const readFile = util.promisify(fs.readFile); +readFile(filename, 'utf-8') + .then(data => JSON.parse(data)) + .then(context => jsonld.toRDF(context, {format: 'application/n-quads'})) + .catch(e => { + if (e?.details?.code) { + console.log(`${e.message} (${e.details.code})`); + } else { + console.log(e.message); + } + process.exit(26); + }); + diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..c7576bc --- /dev/null +++ b/package-lock.json @@ -0,0 +1,233 @@ +{ + "name": "bblocks-postprocess-action", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "jsonld": "^8.3.2" + } + }, + "node_modules/@digitalbazaar/http-client": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/@digitalbazaar/http-client/-/http-client-3.4.1.tgz", + "integrity": "sha512-Ahk1N+s7urkgj7WvvUND5f8GiWEPfUw0D41hdElaqLgu8wZScI8gdI0q+qWw5N1d35x7GCRH2uk9mi+Uzo9M3g==", + "dependencies": { + "ky": "^0.33.3", + "ky-universal": "^0.11.0", + "undici": "^5.21.2" + }, + "engines": { + "node": ">=14.0" + } + }, + "node_modules/@fastify/busboy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", + "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==", + "engines": { + "node": ">=14" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/canonicalize": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/canonicalize/-/canonicalize-1.0.8.tgz", + "integrity": "sha512-0CNTVCLZggSh7bc5VkX5WWPWO+cyZbNd07IHIsSXLia/eAq+r836hgk+8BKoEh7949Mda87VUOitx5OddVj64A==" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/jsonld": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/jsonld/-/jsonld-8.3.2.tgz", + "integrity": "sha512-MwBbq95szLwt8eVQ1Bcfwmgju/Y5P2GdtlHE2ncyfuYjIdEhluUVyj1eudacf1mOkWIoS9GpDBTECqhmq7EOaA==", + "dependencies": { + "@digitalbazaar/http-client": "^3.4.1", + "canonicalize": "^1.0.1", + "lru-cache": "^6.0.0", + "rdf-canonize": "^3.4.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/ky": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/ky/-/ky-0.33.3.tgz", + "integrity": "sha512-CasD9OCEQSFIam2U8efFK81Yeg8vNMTBUqtMOHlrcWQHqUX3HeCl9Dr31u4toV7emlH8Mymk5+9p0lL6mKb/Xw==", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sindresorhus/ky?sponsor=1" + } + }, + "node_modules/ky-universal": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/ky-universal/-/ky-universal-0.11.0.tgz", + "integrity": "sha512-65KyweaWvk+uKKkCrfAf+xqN2/epw1IJDtlyCPxYffFCMR8u1sp2U65NtWpnozYfZxQ6IUzIlvUcw+hQ82U2Xw==", + "dependencies": { + "abort-controller": "^3.0.0", + "node-fetch": "^3.2.10" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sindresorhus/ky-universal?sponsor=1" + }, + "peerDependencies": { + "ky": ">=0.31.4", + "web-streams-polyfill": ">=3.2.1" + }, + "peerDependenciesMeta": { + "web-streams-polyfill": { + "optional": true + } + } + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/rdf-canonize": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/rdf-canonize/-/rdf-canonize-3.4.0.tgz", + "integrity": "sha512-fUeWjrkOO0t1rg7B2fdyDTvngj+9RlUyL92vOdiB7c0FPguWVsniIMjEtHH+meLBO9rzkUlUzBVXgWrjI8P9LA==", + "dependencies": { + "setimmediate": "^1.0.5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" + }, + "node_modules/undici": { + "version": "5.28.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz", + "integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..f366bbc --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "jsonld": "^8.3.2" + } +}