Merge remote-tracking branch 'moat/main'

.gitignore

@@ -67,9 +67,11 @@ __pycache__/
 *.swp
 /.tested.yaml
 /tests/plugins/test.db
+/_test.log
 .tox/
 /*.upload
 /var/
 /.venv
 /venv/
 /wheels/
+/tests/mqtt/plugins/test.db

.gitmodules

@@ -1,12 +1,12 @@
 [submodule "picolibc"]
-	path = TODO/bus/c/picolibc
+	path = ext/picolibc
 	url = https://github.com/M-o-a-T/picolibc.git
 [submodule "lib/serialpacker"]
-	path = TODO/micro/serialpacker
+	path = ext/serialpacker
 	url = https://github.com/M-o-a-T/serialpacker.git
 [submodule "lib/micropython"]
-	path = TODO/micro/micropython
+	path = ext/micropython
 	url = https://github.com/M-o-a-T/micropython.git
 [submodule "lib/mpdb"]
-	path = TODO/micro/mpdb
-	url = https://github.com/bobveringa/mpdb.git
+	path = ext/mpdb
+	url = https://github.com/M-o-a-T/mpdb.git

Makefile

@@ -13,3 +13,12 @@ else
 	@exit 1
 endif
 
+prep:
+	git submodule update --init --recursive
+	make -C ext/micropython/mpy-cross
+	make -C ext/micropython/ports/unix
+	@echo "You might want to do 'make upy-install'"
+
+upy-install: prep
+	sudo cp ext/micropython/mpy-cross/build/mpy-cross /usr/local/bin/
+	sudo cp ext/micropython/ports/unix/build-standard/micropython /usr/local/bin/

TODO/bus/python/distkv_ext/moatbus/client.py

@@ -8,12 +8,23 @@
 import time
 import datetime
 
-from distkv.util import yprint, attrdict, NotGiven, as_service, P, Path, path_eval, attr_args, process_args
+from distkv.util import (
+    yprint,
+    attrdict,
+    NotGiven,
+    as_service,
+    P,
+    Path,
+    path_eval,
+    attr_args,
+    process_args,
+)
 from distkv.obj.command import std_command
 
 from .model import MOATroot
 from moatbus.message import BusMessage
 
+
 @click.group(short_help="Manage MOAT devices.")
 @click.pass_obj
 async def cli(obj):
@@ -32,9 +43,7 @@ async def dump(obj, path):
     res = {}
     path = P(path)
 
-    async for r in obj.client.get_tree(
-        obj.cfg.moatbus.prefix + path, nchain=obj.meta
-    ):
+    async for r in obj.client.get_tree(obj.cfg.moatbus.prefix + path, nchain=obj.meta):
         # pl = len(path) + len(r.path)
         rr = res
         if r.path:
@@ -47,25 +56,26 @@ async def dump(obj, path):
 cmd_bus = std_command(
     cli,
     "bus",
-    aux=(
-        click.option("-t", "--topic", type=P, help="MQTT topic for bus messages"),
-    ),
+    aux=(click.option("-t", "--topic", type=P, help="MQTT topic for bus messages"),),
     sub_name="bus",
     id_name=None,
-    short_help="Manage MoaT buses"
+    short_help="Manage MoaT buses",
 )
 
+
 @cli.command("type", short_help="list connection types/params")
 @click.argument("type_", nargs=-1)
 @click.pass_obj
 def typ_(obj, type_):
     if not type_:
-        type_=[]
+        type_ = []
         print("Known connection types:", file=obj.stdout)
         ext = importlib.import_module("moatbus.backend")
-        for finder, name, ispkg in pkgutil.iter_modules(ext.__path__, ext.__name__ + "."):
+        for finder, name, ispkg in pkgutil.iter_modules(
+            ext.__path__, ext.__name__ + "."
+        ):
             n = name.rsplit(".")[-1]
-            if n[0] == '_':
+            if n[0] == "_":
                 continue
             type_.append(n)
 
@@ -75,21 +85,22 @@ def typ_(obj, type_):
 
         m = importlib.import_module(f"moatbus.backend.{mn}")
         cnt = 0
-        for n,x in m.Handler.PARAMS.items():
+        for n, x in m.Handler.PARAMS.items():
             if not cnt:
-                table.append(("*",mn,m.Handler.short_help))
-            t,i,c,d,m = x
+                table.append(("*", mn, m.Handler.short_help))
+            t, i, c, d, m = x
             tn = "Path" if t is P else t.__name__
-            table.append((n,tn,i))
+            table.append((n, tn, i))
             cnt += 1
         if not cnt:
-            table.append(("*",mn,m.Handler.short_help+"(no params)"))
+            table.append(("*", mn, m.Handler.short_help + "(no params)"))
 
     if table:
         print(tabulate(table, tablefmt="plain", disable_numparse=True), file=obj.stdout)
     elif obj.verbose:
         print("No buses known.", file=sys.stderr)
 
+
 @cmd_bus.command()
 @click.pass_obj
 async def monitor(obj):
@@ -100,8 +111,10 @@ async def monitor(obj):
         print("---", file=obj.stdout)
         async for msg in mon:
             msg["time"] = time.time()
-            msg["_time"] = datetime.datetime.now().isoformat(sep=" ", timespec="milliseconds")
-            mid = msg.data.pop("_id",None)
+            msg["_time"] = datetime.datetime.now().isoformat(
+                sep=" ", timespec="milliseconds"
+            )
+            mid = msg.data.pop("_id", None)
             if mid is not None:
                 msg["_id"] = mid
 
@@ -112,6 +125,7 @@ async def monitor(obj):
             print("---", file=obj.stdout)
             obj.stdout.flush()
 
+
 def set_conn(obj, kw):
     type_ = kw.pop("type_")
     vars_ = kw.pop("vars_")
@@ -125,30 +139,41 @@ def set_conn(obj, kw):
     obj.typ = type_
     obj.params = params
 
+
 cmd_conn = std_command(
     cmd_bus,
     "conn",
     long_name="bus connection",
     id_name=None,
     aux=(
-        click.option("-t", "--type", "type_", type=str, default=None, help="Connection type"),
-        click.option("-h", "--host", type=str, default=None, help="Node this may run on"),
+        click.option(
+            "-t", "--type", "type_", type=str, default=None, help="Connection type"
+        ),
+        click.option(
+            "-h", "--host", type=str, default=None, help="Node this may run on"
+        ),
         attr_args,
     ),
     sub_base="bus",
     sub_name=NotGiven,
     apply=set_conn,
 )
 
+
 @cmd_conn.command()
-@click.option("-f","--force",is_flag=True,help="Force running despite wrong host")
+@click.option("-f", "--force", is_flag=True, help="Force running despite wrong host")
 @click.pass_obj
 async def run(obj, force):
     """Stand-alone task to talk to a single server."""
     from distkv_ext.moatbus.task import gateway
     from distkv_ext.moatbus.model import conn_backend
 
-    if not force and obj.conn.host is not None and obj.client.client_name != obj.conn.host:
-        raise RuntimeError(f"Runs on {obj.conn.host} but this is {obj.client.client_name}")
+    if (
+        not force
+        and obj.conn.host is not None
+        and obj.client.client_name != obj.conn.host
+    ):
+        raise RuntimeError(
+            f"Runs on {obj.conn.host} but this is {obj.client.client_name}"
+        )
     await gateway(obj.conn)
-

TODO/bus/python/distkv_ext/moatbus/config.py

@@ -3,10 +3,8 @@
 CFG = attrdict(
     # Storage path for bus devices
     prefix=Path(".distkv", "moat", "bus"),
-
     # MQTT channel to relay bus messages
     topic=Path("moat", "bus", "data"),
-
     # address assignment processing default
     addr=attrdict(
         timeout=5,

TODO/bus/python/distkv_ext/moatbus/dump.py

@@ -2,8 +2,13 @@
 import random
 from moatbus.backend.mqtt import MqttBusHandler
 
+
 @click.group(short_help="Display MoatBUS messages", invoke_without_command=True)
-@click.option("-i","--ident", help="Identifier for this process. Must be unique. Default is random.")
+@click.option(
+    "-i",
+    "--ident",
+    help="Identifier for this process. Must be unique. Default is random.",
+)
 @click.pass_context
 async def cli(ctx, ident):
     """
@@ -16,7 +21,8 @@ async def cli(ctx, ident):
 
     if ctx.invoked_subcommand is not None:
         return
-    async with MqttBusHandler(id=ident, uri=cfg.server.mqtt['uri'], topic=cfg.moatbus.topic) as M:
+    async with MqttBusHandler(
+        id=ident, uri=cfg.server.mqtt["uri"], topic=cfg.moatbus.topic
+    ) as M:
         async for msg in M:
             print(msg)
-

TODO/bus/python/distkv_ext/moatbus/mock.py

@@ -7,6 +7,7 @@
 See ``tests/test_basic.py`` for code that does.
 
 """
+
 import os
 import trio
 import tempfile

TODO/bus/python/distkv_ext/moatbus/model.py

@@ -1,6 +1,7 @@
 """
 DistKV client data model for MoaT bus devices
 """
+
 import anyio
 
 import asyncclick as click
@@ -9,6 +10,7 @@
 from distkv.util import Path
 from distkv.errors import ErrorRoot
 
+
 class _MOATbase(ClientEntry):
     """
     Forward ``_update_server`` calls to child entries.
@@ -78,7 +80,9 @@ async def _update_value(self):
         if dev is None or dev.bus is None:
             return
 
-        self.val = combine_dict(self.value_or({}, Mapping), self.parent.value_or({}, Mapping))
+        self.val = combine_dict(
+            self.value_or({}, Mapping), self.parent.value_or({}, Mapping)
+        )
 
     async def setup(self, initial=False):
         await super().setup()
@@ -111,6 +115,7 @@ async def _spawn(evt, p, a, k):
         await self.tg.spawn(_spawn, evt, p, a, k)
         await evt.wait()
 
+
 def conn_backend(name):
     from importlib import import_module
 
@@ -121,15 +126,16 @@ def conn_backend(name):
 
 class MOATconn(_MOATbase, AttrClientEntry):
     """Describes one possible connection to this bus."""
+
     typ = None
     params = None
     host = None
 
-    ATTRS = ("typ","params","host")
+    ATTRS = ("typ", "params", "host")
 
-    def __init__(self,*a,**k):
+    def __init__(self, *a, **k):
         self.params = {}
-        super().__init__(*a,**k)
+        super().__init__(*a, **k)
 
     def __str__(self):
         if self.typ is None:
@@ -165,7 +171,9 @@ def backend(self):
                     await process(msg)
         """
         if self.host is not None and self.root.client.client_name != self.host:
-            raise RuntimeError(f"This must run on {self.host}. This is {self.root.client.client_name}")
+            raise RuntimeError(
+                f"This must run on {self.host}. This is {self.root.client.client_name}"
+            )
         return self.handler(self.root.client, **self.params)
 
     @staticmethod
@@ -178,12 +186,16 @@ def check_config(typ, host, params):
             raise click.MissingParameter(param_hint="", param_type="type")
         back = conn_backend(typ)
         if back.need_host and host is None:
-            raise click.MissingParameter(param_hint="", param_type="host", message="Required by this type.")
+            raise click.MissingParameter(
+                param_hint="", param_type="host", message="Required by this type."
+            )
         back.check_config(params)
 
+
 class MOATbus(_MOATbase, AttrClientEntry):
     """Describes one bus, i.e. a collection of clients"""
-    ATTRS = ('topic',)
+
+    ATTRS = ("topic",)
     topic: str = None
 
     @classmethod
@@ -209,7 +221,11 @@ async def set_server(self, server, initial=False):
 
     async def save(self):
         if self.topic is None:
-            raise click.MissingParameter(param_hint="", param_type="topic", message="You need to specify a topic path.")
+            raise click.MissingParameter(
+                param_hint="",
+                param_type="topic",
+                message="You need to specify a topic path.",
+            )
         await super().save()
 
     @property
@@ -220,7 +236,6 @@ def repr(self):
         return res
 
 
-
 class MOATbuses(_MOATbase):
     @classmethod
     def child_type(cls, name):