Merge pull request #316 from bluesky/dtype_numpy

Dtype numpy

.github/actions/install_requirements/action.yml

@@ -12,7 +12,7 @@ inputs:
     default: "3.x"
   jsonschema_version:
     description: version of the jsonschema pip package to install
-    default: 3
+    default: 4
 
 runs:
   using: composite

.github/workflows/code.yml

@@ -40,7 +40,7 @@ jobs:
           - os: "ubuntu-latest"
             python: "3.8"
             install: ".[dev]"
-            jsonschema: 3
+            jsonschema: 4
 
     runs-on: ${{ matrix.os }}
     env:

event_model/__init__.py

@@ -1801,18 +1801,20 @@ class MismatchedDataKeys(InvalidData):
 
 def _is_array(checker, instance):
     return (
-        jsonschema.validators.Draft7Validator.TYPE_CHECKER.is_type(instance, "array")
+        jsonschema.validators.Draft202012Validator.TYPE_CHECKER.is_type(
+            instance, "array"
+        )
         or isinstance(instance, tuple)
         or hasattr(instance, "__array__")
     )
 
 
-_array_type_checker = jsonschema.validators.Draft7Validator.TYPE_CHECKER.redefine(
+_array_type_checker = jsonschema.validators.Draft202012Validator.TYPE_CHECKER.redefine(
     "array", _is_array
 )
 
 _Validator = jsonschema.validators.extend(
-    jsonschema.validators.Draft7Validator, type_checker=_array_type_checker
+    jsonschema.validators.Draft202012Validator, type_checker=_array_type_checker
 )
 
 schema_validators = {

event_model/documents/event_descriptor.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from typing_extensions import Annotated, Literal, NotRequired, TypedDict
 
@@ -24,6 +24,17 @@ class Limits(TypedDict):
     alarm: NotRequired[Annotated[LimitsRange, Field(description="Alarm limits.")]]
 
 
+_ConstrainedDtype = Annotated[
+    str,
+    Field(
+        description="A numpy dtype e.g `<U9`, `<f16`",
+        pattern="[|<>][tbiufcmMOSUV][0-9]+",
+    ),
+]
+
+_ConstrainedDtypeNpStructure = List[Tuple[str, _ConstrainedDtype]]
+
+
 class DataKey(TypedDict):
     """Describes the objects in the data property of Event documents"""
 
@@ -42,7 +53,23 @@ class DataKey(TypedDict):
     ]
     dtype: Annotated[
         Dtype,
-        Field(description="The type of the data in the event."),
+        Field(
+            description=(
+                "The type of the data in the event, given as a broad "
+                "JSON schema type."
+            )
+        ),
+    ]
+    dtype_numpy: NotRequired[
+        Annotated[
+            Union[_ConstrainedDtype, _ConstrainedDtypeNpStructure],
+            Field(
+                description=(
+                    "The type of the data in the event, given as a "
+                    "numpy dtype string (or, for structured dtypes, array)."
+                )
+            ),
+        ]
     ]
     external: NotRequired[
         Annotated[

event_model/schemas/event_descriptor.json

@@ -50,7 +50,7 @@
                 },
                 "dtype": {
                     "title": "Dtype",
-                    "description": "The type of the data in the event.",
+                    "description": "The type of the data in the event, given as a broad JSON schema type.",
                     "type": "string",
                     "enum": [
                         "string",
@@ -60,6 +60,35 @@
                         "integer"
                     ]
                 },
+                "dtype_numpy": {
+                    "title": "Dtype Numpy",
+                    "description": "The type of the data in the event, given as a numpy dtype string (or, for structured dtypes, array).",
+                    "anyOf": [
+                        {
+                            "description": "A numpy dtype e.g `<U9`, `<f16`",
+                            "pattern": "[|<>][tbiufcmMOSUV][0-9]+",
+                            "type": "string"
+                        },
+                        {
+                            "items": {
+                                "maxItems": 2,
+                                "minItems": 2,
+                                "prefixItems": [
+                                    {
+                                        "type": "string"
+                                    },
+                                    {
+                                        "description": "A numpy dtype e.g `<U9`, `<f16`",
+                                        "pattern": "[|<>][tbiufcmMOSUV][0-9]+",
+                                        "type": "string"
+                                    }
+                                ],
+                                "type": "array"
+                            },
+                            "type": "array"
+                        }
+                    ]
+                },
                 "external": {
                     "title": "External",
                     "description": "Where the data is stored if it is stored external to the events",

event_model/tests/test_auth.py

@@ -73,3 +73,62 @@ def test_dots_not_allowed_in_keys():
     doc.update({".b": "c"})
     with pytest.raises(jsonschema.ValidationError):
         schema_validators[DocumentNames.stop].validate(doc)
+
+
+@pytest.mark.parametrize(
+    "dtype_numpy",
+    [
+        "Z",
+        "i",
+        "i4",
+        "4i",
+        "i>4",
+        ">i",
+        ("some_str_1", "<u4"),
+        [("some_str_1", "<u4"), ("some_str", "Z")],
+    ],
+)
+def test_bad_numpy_datakeys(dtype_numpy):
+    descriptor = {
+        "time": 0,
+        "uid": new_uid(),
+        "data_keys": {
+            "a": {
+                "source": "",
+                "dtype": "number",
+                "shape": [],
+                "dtype_numpy": dtype_numpy,
+            }
+        },
+        "run_start": new_uid(),
+    }
+    with pytest.raises(
+        jsonschema.ValidationError,
+    ):
+        schema_validators[DocumentNames.descriptor].validate(descriptor)
+
+
+@pytest.mark.parametrize(
+    "dtype_numpy",
+    [
+        ">u4",
+        "<u4",
+        [("some_str_1", "<u4"), ("some_str", "<u4")],
+        [("some_str_1", "<u4"), ("some_str", ">u4")],
+    ],
+)
+def test_good_numpy_datakeys(dtype_numpy):
+    descriptor = {
+        "time": 0,
+        "uid": new_uid(),
+        "data_keys": {
+            "a": {
+                "source": "",
+                "dtype": "number",
+                "shape": [],
+                "dtype_numpy": dtype_numpy,
+            }
+        },
+        "run_start": new_uid(),
+    }
+    schema_validators[DocumentNames.descriptor].validate(descriptor)

pyproject.toml

@@ -16,7 +16,7 @@ classifiers = [
 description = "Data model used by the bluesky ecosystem"
 dependencies = [
     "importlib-resources",
-    "jsonschema>=3",
+    "jsonschema>=4",
     "numpy",
     "typing_extensions"
 ]