Implement legacy & modern plugin method calls over WS

This version builds fine and runs all of the 14 plugins I have installed perfectly, so we're really close to having this done.

.gitignore

@@ -159,8 +159,8 @@ backend/static
 .vscode/settings.json
 
 # plugins folder for local launches
-plugins/*
+/plugins/*
 act/.directory
 act/artifacts/*
 bin/act
-settings/
+/settings/

.vscode/tasks.json

@@ -37,8 +37,11 @@
             "label": "dependencies",
             "type": "shell",
             "group": "none",
+            "dependsOn": [
+                "deploy"
+            ],
             "detail": "Check for local runs, create a plugins folder",
-            "command": "rsync -azp --rsh='ssh -p ${config:deckport} ${config:deckkey}' backend/pyproject.toml backend/poetry.lock deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader && ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'python -m ensurepip && python -m pip install --upgrade poetry && cd ${config:deckdir}/homebrew/dev/pluginloader/backend && python -m poetry install'",
+            "command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'python -m ensurepip && python -m pip install --user --upgrade poetry && cd ${config:deckdir}/homebrew/dev/pluginloader/backend && python -m poetry install'",
             "problemMatcher": []
         },
         {
@@ -105,7 +108,7 @@
             "detail": "Deploy dev PluginLoader to deck",
             "type": "shell",
             "group": "none",
-            "command": "rsync -azp --delete --rsh='ssh -p ${config:deckport} ${config:deckkey}' --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='backend/decky_loader/__pycache__/' --exclude='.gitignore' . deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader",
+            "command": "rsync -azp --delete --rsh='ssh -p ${config:deckport} ${config:deckkey}' --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='backend/**/__pycache__/' --exclude='.gitignore' . deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader",
             "problemMatcher": []
         },
         // RUN
@@ -117,7 +120,7 @@
             "dependsOn": [
                 "checkforsettings"
             ],
-            "command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'export PLUGIN_PATH=${config:deckdir}/homebrew/dev/plugins; export CHOWN_PLUGIN_PATH=0; export LOG_LEVEL=DEBUG; cd ${config:deckdir}/homebrew/services; echo '${config:deckpass}' | sudo -SE python3 ${config:deckdir}/homebrew/dev/pluginloader/backend/main.py'",
+            "command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'export PATH=${config:deckdir}/.local/bin:$PATH; export PLUGIN_PATH=${config:deckdir}/homebrew/dev/plugins; export CHOWN_PLUGIN_PATH=0; export LOG_LEVEL=DEBUG; cd ${config:deckdir}/homebrew/dev/pluginloader/backend; echo '${config:deckpass}' | sudo -SE poetry run sh -c \"cd ${config:deckdir}/homebrew/services; python3 ${config:deckdir}/homebrew/dev/pluginloader/backend/main.py\"'",
             "problemMatcher": []
         },
         {
@@ -181,7 +184,8 @@
                 "buildall",
                 "createfolders",
                 "dependencies",
-                "deploy",
+                // dependencies runs deploy already
+                // "deploy",
                 "runpydeck"
             ],
             "problemMatcher": []
@@ -190,7 +194,7 @@
             "label": "act",
             "type": "shell",
             "group": "none",
-            "detail": "Run the act thing",
+            "detail": "Build release artifact using local CI",
             "command": "./act/run-act.sh release",
             "problemMatcher": []
         }

backend/decky_loader/helpers.py

@@ -34,7 +34,7 @@ def get_csrf_token():
 
 @middleware
 async def csrf_middleware(request: Request, handler: Handler):
-    if str(request.method) == "OPTIONS" or request.headers.get('Authentication') == csrf_token or str(request.rel_url) == "/auth/token" or str(request.rel_url).startswith("/plugins/load_main/") or str(request.rel_url).startswith("/static/") or str(request.rel_url).startswith("/steam_resource/") or str(request.rel_url).startswith("/frontend/") or assets_regex.match(str(request.rel_url)) or frontend_regex.match(str(request.rel_url)):
+    if str(request.method) == "OPTIONS" or request.headers.get('Authentication') == csrf_token or str(request.rel_url) == "/auth/token" or str(request.rel_url).startswith("/plugins/load_main/") or str(request.rel_url).startswith("/static/") or str(request.rel_url).startswith("/steam_resource/") or str(request.rel_url).startswith("/frontend/") or str(request.rel_url.path) == "/ws" or assets_regex.match(str(request.rel_url)) or frontend_regex.match(str(request.rel_url)):
         return await handler(request)
     return Response(text='Forbidden', status=403)
 

backend/decky_loader/loader.py

@@ -4,15 +4,15 @@
 from logging import getLogger
 from os import listdir, path
 from pathlib import Path
-from traceback import print_exc
-from typing import Any, Tuple
+from traceback import format_exc, print_exc
+from typing import Any, Tuple, Dict
 
 from aiohttp import web
 from os.path import exists
 from watchdog.events import RegexMatchingEventHandler, DirCreatedEvent, DirModifiedEvent, FileCreatedEvent, FileModifiedEvent # type: ignore
 from watchdog.observers import Observer # type: ignore
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, List
 if TYPE_CHECKING:
     from .main import PluginManager
 
@@ -92,13 +92,15 @@ def __init__(self, server_instance: PluginManager, ws: WSRouter, plugin_path: st
         server_instance.web_app.add_routes([
             web.get("/frontend/{path:.*}", self.handle_frontend_assets),
             web.get("/locales/{path:.*}", self.handle_frontend_locales),
-            web.get("/plugins", self.get_plugins),
             web.get("/plugins/{plugin_name}/frontend_bundle", self.handle_frontend_bundle),
-            web.post("/plugins/{plugin_name}/methods/{method_name}", self.handle_plugin_method_call),
             web.get("/plugins/{plugin_name}/assets/{path:.*}", self.handle_plugin_frontend_assets),
-            web.post("/plugins/{plugin_name}/reload", self.handle_backend_reload_request)
         ])
 
+        server_instance.ws.add_route("loader/get_plugins", self.get_plugins)
+        server_instance.ws.add_route("loader/reload_plugin", self.handle_plugin_backend_reload)
+        server_instance.ws.add_route("loader/call_plugin_method", self.handle_plugin_method_call)
+        server_instance.ws.add_route("loader/call_legacy_plugin_method", self.handle_plugin_method_call_legacy)
+
     async def enable_reload_wait(self):
         if self.live_reload:
             await sleep(10)
@@ -119,9 +121,9 @@ async def handle_frontend_locales(self, request: web.Request):
             self.logger.info(f"Language {req_lang} not available, returning an empty dictionary")
             return web.json_response(data={}, headers={"Cache-Control": "no-cache"})
 
-    async def get_plugins(self, request: web.Request):
+    async def get_plugins(self):
         plugins = list(self.plugins.values())
-        return web.json_response([{"name": str(i), "version": i.version} for i in plugins])
+        return [{"name": str(i), "version": i.version} for i in plugins]
 
     async def handle_plugin_frontend_assets(self, request: web.Request):
         plugin = self.plugins[request.match_info["plugin_name"]]
@@ -173,29 +175,31 @@ async def handle_reloads(self):
             args = await self.reload_queue.get()
             self.import_plugin(*args) # type: ignore
 
-    async def handle_plugin_method_call(self, request: web.Request):
-        res = {}
-        plugin = self.plugins[request.match_info["plugin_name"]]
-        method_name = request.match_info["method_name"]
-        try:
-            method_info = await request.json()
-            args: Any = method_info["args"]
-        except JSONDecodeError:
-            args = {}
+    async def handle_plugin_method_call_legacy(self, plugin_name: str, method_name: str, kwargs: Dict[Any, Any]):
+        res: Dict[Any, Any] = {}
+        plugin = self.plugins[plugin_name]
         try:
           if method_name.startswith("_"):
-              raise RuntimeError("Tried to call private method")
-          res["result"] = await plugin.execute_method(method_name, args)
+              raise RuntimeError(f"Plugin {plugin.name} tried to call private method {method_name}")
+          res["result"] = await plugin.execute_legacy_method(method_name, kwargs)
           res["success"] = True
         except Exception as e:
             res["result"] = str(e)
             res["success"] = False
-        return web.json_response(res)
+        return res
 
-    async def handle_backend_reload_request(self, request: web.Request):
-        plugin_name : str = request.match_info["plugin_name"]
+    async def handle_plugin_method_call(self, plugin_name: str, method_name: str, *args: List[Any]):
         plugin = self.plugins[plugin_name]
+        try:
+          if method_name.startswith("_"):
+              raise RuntimeError(f"Plugin {plugin.name} tried to call private method {method_name}")
+          result = await plugin.execute_method(method_name, *args)
+        except Exception as e:
+            self.logger.error(f"Method {method_name} of plugin {plugin.name} failed with the following exception:\n{format_exc()}")
+            raise e # throw again to pass the error to the frontend
+        return result
 
-        await self.reload_queue.put((plugin.file, plugin.plugin_directory))
+    async def handle_plugin_backend_reload(self, plugin_name: str):
+        plugin = self.plugins[plugin_name]
 
-        return web.Response(status=200)
+        await self.reload_queue.put((plugin.file, plugin.plugin_directory))

backend/decky_loader/main.py

@@ -1,7 +1,6 @@
 # Change PyInstaller files permissions
 import sys
 from typing import Dict
-from wsrouter import WSRouter
 from .localplatform.localplatform import (chmod, chown, service_stop, service_start,
                             ON_WINDOWS, get_log_level, get_live_reload, 
                             get_server_port, get_server_host, get_chown_plugin_path,
@@ -32,6 +31,7 @@
 from .updater import Updater
 from .utilities import Utilities
 from .customtypes import UserType
+from .wsrouter import WSRouter
 
 
 basicConfig(
@@ -102,7 +102,6 @@ async def load_plugins(self):
         # await self.wait_for_server()
         logger.debug("Loading plugins")
         self.plugin_loader.import_plugins()
-        # await inject_to_tab("SP", "window.syncDeckyPlugins();")
         if self.settings.getSetting("pluginOrder", None) == None:
           self.settings.setSetting("pluginOrder", list(self.plugin_loader.plugins.keys()))
           logger.debug("Did not find pluginOrder setting, set it to default")

backend/decky_plugin.py → backend/decky_loader/plugin/imports/decky.py

@@ -19,6 +19,8 @@
 import logging
 import time
 
+from typing import Dict, Any
+
 """
 Constants
 """
@@ -207,3 +209,13 @@ def migrate_logs(*files_or_directories: str) -> dict[str, str]:
 """The main plugin logger writing to `DECKY_PLUGIN_LOG`."""
 
 logger.setLevel(logging.INFO)
+
+"""
+Event handling
+"""
+# TODO better docstring im lazy
+async def emit_message(message: Dict[Any, Any]) -> None:
+    """
+    Send a message to the frontend.
+    """
+    pass

backend/decky_plugin.pyi → backend/decky_loader/plugin/imports/decky.pyi

@@ -16,6 +16,8 @@ __version__ = '0.1.0'
 
 import logging
 
+from typing import Dict, Any
+
 """
 Constants
 """
@@ -171,3 +173,12 @@ Logging
 
 logger: logging.Logger
 """The main plugin logger writing to `DECKY_PLUGIN_LOG`."""
+
+"""
+Event handling
+"""
+# TODO better docstring im lazy
+async def emit_message(message: Dict[Any, Any]) -> None:
+    """
+    Send a message to the frontend.
+    """

backend/decky_loader/plugin/plugin.py

@@ -4,11 +4,12 @@
 from os import path
 from multiprocessing import Process
 
+
 from .sandboxed_plugin import SandboxedPlugin
 from .method_call_request import MethodCallRequest
 from ..localplatform.localsocket import LocalSocket
 
-from typing import Any, Callable, Coroutine, Dict
+from typing import Any, Callable, Coroutine, Dict, List
 
 class PluginWrapper:
     def __init__(self, file: str, plugin_directory: str, plugin_path: str) -> None:
@@ -39,6 +40,8 @@ def __init__(self, file: str, plugin_directory: str, plugin_path: str) -> None:
 
         self.emitted_message_callback: Callable[[Dict[Any, Any]], Coroutine[Any, Any, Any]]
 
+        self.legacy_method_warning = False
+
     def __str__(self) -> str:
         return self.name
 
@@ -58,13 +61,27 @@ async def _response_listener(self):
     def set_emitted_message_callback(self, callback: Callable[[Dict[Any, Any]], Coroutine[Any, Any, Any]]):
         self.emitted_message_callback = callback
 
-    async def execute_method(self, method_name: str, kwargs: Dict[Any, Any]):
+    async def execute_legacy_method(self, method_name: str, kwargs: Dict[Any, Any]):
+        if not self.legacy_method_warning:
+            self.legacy_method_warning = True
+            self.log.warn(f"Plugin {self.name} is using legacy method calls. This will be removed in a future release.")
+        if self.passive:
+            raise RuntimeError("This plugin is passive (aka does not implement main.py)")
+
+        request = MethodCallRequest()
+        await self._socket.get_socket_connection()
+        await self._socket.write_single_line(dumps({ "method": method_name, "args": kwargs, "id": request.id, "legacy": True }, ensure_ascii=False))
+        self._method_call_requests[request.id] = request
+
+        return await request.wait_for_result()
+
+    async def execute_method(self, method_name: str, args: List[Any]):
         if self.passive:
             raise RuntimeError("This plugin is passive (aka does not implement main.py)")
 
         request = MethodCallRequest()
         await self._socket.get_socket_connection()
-        await self._socket.write_single_line(dumps({ "method": method_name, "args": kwargs, "id": request.id }, ensure_ascii=False))
+        await self._socket.write_single_line(dumps({ "method": method_name, "args": args, "id": request.id }, ensure_ascii=False))
         self._method_call_requests[request.id] = request
 
         return await request.wait_for_result()

backend/decky_loader/plugin/sandboxed_plugin.py

@@ -77,17 +77,28 @@ def initialize(self, socket: LocalSocket):
             keys = [key for key in sysmodules if key.startswith("decky_loader.")]
             for key in keys:
                 sysmodules[key.replace("decky_loader.", "")] = sysmodules[key]
+
+            from .imports import decky
+            async def emit_message(message: Dict[Any, Any]):
+                await self._socket.write_single_line_server(dumps({
+                    "id": "0",
+                    "payload": message
+                }))
+            # copy the docstring over so we don't have to duplicate it
+            emit_message.__doc__ = decky.emit_message.__doc__
+            decky.emit_message = emit_message
+            sysmodules["decky"] = decky
+            # provided for compatibility
+            sysmodules["decky_plugin"] = decky
 
             spec = spec_from_file_location("_", self.file)
             assert spec is not None
             module = module_from_spec(spec)
             assert spec.loader is not None
             spec.loader.exec_module(module)
+            # TODO fix self weirdness once plugin.json versioning is done. need this before WS release!
             self.Plugin = module.Plugin
 
-            setattr(self.Plugin, "emit_message", self.emit_message)
-            #TODO: Find how to put emit_message on global namespace so it doesn't pollute Plugin
-
             if hasattr(self.Plugin, "_migration"):
                 get_event_loop().run_until_complete(self.Plugin._migration(self.Plugin))
             if hasattr(self.Plugin, "_main"):
@@ -124,15 +135,14 @@ async def on_new_message(self, message : str) -> str|None:
 
         d: SocketResponseDict = {"res": None, "success": True, "id": data["id"]}
         try:
-            d["res"] = await getattr(self.Plugin, data["method"])(self.Plugin, **data["args"])
+            if data["legacy"]:
+                # Legacy kwargs
+                d["res"] = await getattr(self.Plugin, data["method"])(self.Plugin, **data["args"])
+            else:
+                # New args
+                d["res"] = await getattr(self.Plugin, data["method"])(self.Plugin, *data["args"])
         except Exception as e:
             d["res"] = str(e)
             d["success"] = False
         finally:
-            return dumps(d, ensure_ascii=False)
-
-    async def emit_message(self, message: Dict[Any, Any]):
-        await self._socket.write_single_line_server(dumps({
-            "id": "0",
-            "payload": message
-        }))
+            return dumps(d, ensure_ascii=False)