Add tests.yaml to support test resources by reference

ogc/bblocks/examples-schema.yaml

@@ -1,5 +1,5 @@
 "$schema": https://json-schema.org/draft/2020-12/schema
-title: OGC Building Blocks Register metadata schema
+title: OGC Building Blocks examples schema
 type: array
 items:
   type: object

ogc/bblocks/extra-tests-schema.yaml

@@ -0,0 +1,23 @@
+"$schema": https://json-schema.org/draft/2020-12/schema
+title: OGC Building Blocks extra tests schema
+type: array
+items:
+  type: object
+  required:
+    - ref
+  properties:
+    ref:
+      description: |
+        A reference to a URL or to a filename (relative to the examples.yaml file) with the contents of
+        this test, as an alternative to inlining them in the 'code' property.
+      type: string
+      format: uri-reference
+    require-fail:
+      description: Whether this test should return an error in order to succeed
+      type: boolean
+      default: false
+    output-filename:
+      description: |
+        Name for the output file name that will be used when generating uplifted versions of the test resource.
+        If not provided, the file name from "ref" will be used.
+      type: string

ogc/bblocks/util.py

@@ -11,8 +11,8 @@
 from collections import deque
 from functools import lru_cache
 from pathlib import Path
-from typing import Any, Sequence, Callable, AnyStr
-from urllib.parse import urljoin
+from typing import Any, Sequence, Callable, AnyStr, Generator, cast
+from urllib.parse import urljoin, urlparse
 
 import jsonschema
 import networkx as nx
@@ -149,7 +149,7 @@ def _load_examples(self):
             try:
                 jsonschema.validate(examples, get_schema('examples'))
             except Exception as e:
-                raise BuildingBlockError('Error validating building block examples') from e
+                raise BuildingBlockError('Error validating building block examples (examples.yaml)') from e
 
             for example in examples:
                 for snippet in example.get('snippets', ()):
@@ -199,6 +199,26 @@ def jsonld_context_contents(self):
             self._lazy_properties['jsonld_context_contents'] = load_file(self.jsonld_context)
         return self._lazy_properties['jsonld_context_contents']
 
+    def get_extra_test_resources(self) -> Generator[dict, None, None]:
+        extra_tests_file = self.files_path / 'tests.yaml'
+        if extra_tests_file.is_file():
+            extra_tests: list[dict] = cast(list[dict], load_yaml(extra_tests_file))
+            try:
+                jsonschema.validate(extra_tests, get_schema('extra-tests'))
+            except Exception as e:
+                raise BuildingBlockError('Error validating extra tests (tests.yaml)') from e
+
+            for test in extra_tests:
+                ref = self.resolve_file(test['ref'])
+                test['ref'] = ref
+                test['contents'] = load_file(ref)
+                if not test.get('output-filename'):
+                    if isinstance(ref, Path):
+                        test['output-filename'] = ref.name
+                    else:
+                        test['output-filename'] = os.path.basename(urlparse(ref).path)
+                yield test
+
     def resolve_file(self, fn_or_url):
         if isinstance(fn_or_url, Path) or (isinstance(fn_or_url, str) and not is_url(fn_or_url)):
             # assume file

ogc/bblocks/validate.py

@@ -76,6 +76,7 @@ class ValidationItemSource:
     snippet_index: int | None = None
     language: str | None = None
     require_fail: bool = False
+    source_url: str | None = None
 
 
 @dataclasses.dataclass
@@ -180,6 +181,8 @@ def report_to_dict(bblock: BuildingBlock,
                     source['snippetIndex'] = item.source.snippet_index
             if item.source.language:
                 source['language'] = item.source.language
+            if item.source.source_url:
+                source['sourceUrl'] = item.source.source_url
 
             sections = []
             for section_enum, entries in item.sections.items():
@@ -267,35 +270,48 @@ def _validate_resource(bblock: BuildingBlock,
                        base_uri: str | None = None,
                        schema_ref: str | None = None,
                        shacl_closure_files: list[str | Path] | None = None,
-                       shacl_closure: Graph | None = None) -> ValidationReportItem:
+                       shacl_closure: Graph | None = None,
+                       require_fail: bool | None = None,
+                       resource_url: str | None = None,
+                       example_index: tuple[int, int] | None = None) -> ValidationReportItem:
 
-    require_fail = filename.stem.endswith('-fail') and not resource_contents
-    if resource_contents:
+    if require_fail is None:
+        require_fail = filename.stem.endswith('-fail') and not example_index
+
+    if resource_url and not is_url(resource_url):
+        resource_url = None
+
+    if example_index:
         example_idx, snippet_idx = re.match(r'example_(\d+)_(\d+).*', filename.name).groups()
         source = ValidationItemSource(
             type=ValidationItemSourceType.EXAMPLE,
             filename=output_filename,
             example_index=int(example_idx),
             snippet_index=int(snippet_idx),
             language=filename.suffix[1:],
+            source_url=resource_url,
         )
     else:
         source = ValidationItemSource(
             type=ValidationItemSourceType.TEST_RESOURCE,
             filename=filename,
             language=filename.suffix[1:],
-            require_fail=filename.stem.endswith('-fail'),
+            require_fail=require_fail,
+            source_url=resource_url,
         )
     report = ValidationReportItem(source)
 
     def validate_inner():
         json_doc = None
         graph = None
 
-        if filename.suffix in ('.json', '.jsonld'):
+        if filename.suffix in ('.json', '.jsonld', '.yaml', '.yml'):
             try:
                 if resource_contents:
-                    json_doc = load_yaml(content=resource_contents)
+                    if filename.suffix.startswith('.json'):
+                        json_doc = json.loads(resource_contents)
+                    else:
+                        json_doc = load_yaml(content=resource_contents)
                     if filename.name == output_filename.name:
                         using_fn = filename.name
                     else:
@@ -314,7 +330,7 @@ def validate_inner():
             except MarkedYAMLError as e:
                 report.add_entry(ValidationReportEntry(
                     section=ValidationReportSection.JSON_SCHEMA,
-                    message=f"Error parsing JSON example: {str(e)} "
+                    message=f"Error parsing YAML example: {str(e)} "
                             f"on or near line {e.context_mark.line + 1} "
                             f"column {e.context_mark.column + 1}",
                     is_error=True,
@@ -325,6 +341,20 @@ def validate_inner():
                     }
                 ))
                 return
+            except JSONDecodeError as e:
+                report.add_entry(ValidationReportEntry(
+                    section=ValidationReportSection.JSON_SCHEMA,
+                    message=f"Error parsing JSON example: {str(e)} "
+                            f"on or near line {e.lineno} "
+                            f"column {e.colno}",
+                    is_error=True,
+                    payload={
+                        'exception': e.__class__.__qualname__,
+                        'line': e.lineno,
+                        'col': e.colno,
+                    }
+                ))
+                return
 
             if '@graph' in json_doc:
                 json_doc = json_doc['@graph']
@@ -420,7 +450,7 @@ def validate_inner():
 
         elif filename.suffix == '.ttl':
             try:
-                if resource_contents:
+                if example_index:
                     report.add_entry(ValidationReportEntry(
                         section=ValidationReportSection.FILES,
                         message=f'Using {filename.name} from examples',
@@ -512,7 +542,7 @@ def validate_inner():
                 json_doc = {'$schema': schema_url, **json_doc}
 
             if resource_contents:
-                # This is an example, write it to disk
+                # This is an example or a ref, write it to disk
                 with open(output_filename, 'w') as f:
                     json.dump(json_doc, f, indent=2)
 
@@ -725,6 +755,30 @@ def validate_test_resources(bblock: BuildingBlock,
             final_result = not test_result.failed and final_result
             test_count += 1
 
+    for extra_test_resource in bblock.get_extra_test_resources():
+        if not re.search(r'\.(json(ld)?|ttl)$', extra_test_resource['output-filename']):
+            continue
+        fn = bblock.files_path / 'tests' / extra_test_resource['output-filename']
+        output_fn = output_dir / fn.name
+        output_base_filenames.add(fn.stem)
+
+        test_result = _validate_resource(
+            bblock, fn, output_fn,
+            resource_contents=extra_test_resource['contents'],
+            schema_validator=schema_validator,
+            jsonld_context=jsonld_context,
+            jsonld_url=jsonld_url,
+            shacl_graphs=shacl_graphs,
+            json_error=json_error,
+            shacl_errors=shacl_errors,
+            shacl_closure=bblock_shacl_closure,
+            require_fail=extra_test_resource.get('require-fail', False),
+            resource_url=extra_test_resource['ref'] if isinstance(extra_test_resource['ref'], str) else None,
+        )
+        all_results.append(test_result)
+        final_result = not test_result.failed and final_result
+        test_count += 1
+
     # Examples
     if bblock.examples:
         for example_id, example in enumerate(bblock.examples):
@@ -778,11 +832,12 @@ def validate_test_resources(bblock: BuildingBlock,
                     snippet_shacl_closure: list[str | Path] = snippet.get('shacl-closure')
                     if snippet_shacl_closure:
                         snippet_shacl_closure = [c if is_url(c) else bblock.files_path.joinpath(c)
-                                         for c in snippet_shacl_closure]
+                                                 for c in snippet_shacl_closure]
 
                     example_result = _validate_resource(
                         bblock, fn, output_fn,
                         resource_contents=code,
+                        example_index=(example_id + 1, snippet_id + 1),
                         schema_url=schema_url,
                         schema_validator=snippet_schema_validator,
                         jsonld_context=jsonld_context,
@@ -793,7 +848,9 @@ def validate_test_resources(bblock: BuildingBlock,
                         base_uri=snippet.get('base-uri', example_base_uri),
                         schema_ref=snippet.get('schema-ref'),
                         shacl_closure_files=snippet_shacl_closure,
-                        shacl_closure=bblock_shacl_closure)
+                        shacl_closure=bblock_shacl_closure,
+                        resource_url=snippet.get('ref'),
+                    )
                     all_results.append(example_result)
                     final_result = final_result and not example_result.failed
                     for file_format, file_data in example_result.uplifted_files.items():