refactor: the big packaging/versioning rework

- move some files around so everything we need is in a single Python package
- build that Python package with Poetry
- write a shared spec file for PyInstaller builds
- use the metadata from the package to determine versions

.github/workflows/build-win.yml

@@ -25,12 +25,14 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "3.11.4"
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: false
 
     - name: Install Python dependencies ⬇️
-      run: |
-        python -m pip install --upgrade pip
-        pip install pyinstaller==5.13.0
-        pip install -r requirements.txt
+      run: C:\Users\runneradmin\.local\bin\poetry install --no-interaction
 
     - name: Install JS dependencies ⬇️
       working-directory: ./frontend
@@ -43,11 +45,11 @@ jobs:
       run: pnpm run build
 
     - name: Build Python Backend 🛠️
-      run: pyinstaller --noconfirm --onefile --name "PluginLoader" --add-data "./backend/static;/static" --add-data "./backend/locales;/locales" --add-data "./backend/legacy;/legacy" --add-data "./plugin;/plugin" --hidden-import=sqlite3 ./backend/main.py
+      run: C:\Users\runneradmin\.local\bin\poetry run pyinstaller decky_loader.spec
 
     - name: Build Python Backend (noconsole) 🛠️
-      run: pyinstaller --noconfirm --noconsole --onefile --name "PluginLoader_noconsole" --add-data "./backend/static;/static" --add-data "./backend/locales;/locales" --add-data "./backend/legacy;/legacy" --add-data "./plugin;/plugin" --hidden-import=sqlite3 ./backend/main.py
-        
+      run: $env:DECKY_NOCONSOLE = 1; C:\Users\runneradmin\.local\bin\poetry run pyinstaller decky_loader.spec
+
     - name: Upload package artifact ⬆️
       uses: actions/upload-artifact@v3
       with:

.github/workflows/build.yml

@@ -69,11 +69,13 @@ jobs:
         sudo cp /usr/lib/libsqlite3.so.0.8.6 /usr/lib/x86_64-linux-gnu/ &&
         rm -r /tmp/sqlite-autoconf-3420000
 
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: false
+
     - name: Install Python dependencies ⬇️
-      run: |
-        python -m pip install --upgrade pip
-        pip install pyinstaller==5.13.0
-        [ -f requirements.txt ] && pip install -r requirements.txt
+      run: poetry install --no-interaction
 
     - name: Install JS dependencies ⬇️
       working-directory: ./frontend
@@ -86,7 +88,7 @@ jobs:
       run: pnpm run build
 
     - name: Build Python Backend 🛠️
-      run: pyinstaller --noconfirm --onefile --name "PluginLoader" --add-data ./backend/static:/static --add-data ./backend/locales:/locales --add-data ./backend/legacy:/legacy --add-data ./plugin:/plugin --hidden-import=sqlite3 ./backend/*.py
+      run: pyinstaller decky_loader.spec
 
     - name: Upload package artifact ⬆️
       if: ${{ !env.ACT }}

.github/workflows/edit-check.yml

@@ -22,23 +22,21 @@ jobs:
       with:
         separator: ","
         files: |
-          plugin/*
+          decky_loader/plugin_api/*
           
     - name: Is stub changed
       id: changed-stub
       run: |
         STUB_CHANGED="false"
-        PATHS=(plugin plugin/decky_plugin.pyi)
+        PATHS=(decky_loader decky_loader/plugin_api/decky_plugin.pyi)
         SHA=${{ github.sha }}
         SHA_PREV=HEAD^
         FILES=$(git diff $SHA_PREV..$SHA --name-only -- ${PATHS[@]} | jq -Rsc 'split("\n")[:-1] | join (",")')
-        if [[ "$FILES" == *"plugin/decky_plugin.pyi"* ]]; then
-          $STUB_CHANGED="true"
+        if [[ "$FILES" == *"decky_loader/plugin_api/decky_plugin.pyi"* ]]; then
+          STUB_CHANGED="true"
           echo "Stub has changed, pushing updated stub"
         else
           echo "Stub has not changed, exiting."
-          echo "has_changed=$STUB_CHANGED" >> $GITHUB_OUTPUT
-          exit 0
         fi
         echo "has_changed=$STUB_CHANGED" >> $GITHUB_OUTPUT
 
@@ -48,7 +46,7 @@ jobs:
       env:
         API_TOKEN_GITHUB: ${{ secrets.GITHUB_TOKEN }}
       with:
-        source_file: 'plugin/decky_plugin.pyi'
+        source_file: 'decky_loader/plugin_api/decky_plugin.pyi'
         destination_repo: 'SteamDeckHomebrew/decky-plugin-template'
         user_email: '[email protected]'
         user_name: 'TrainDoctor'

.gitignore

@@ -29,7 +29,7 @@ MANIFEST
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
-*.spec
+dist/
 
 # Installer logs
 pip-log.txt
@@ -152,7 +152,7 @@ dmypy.json
 cython_debug/
 
 # static files are built
-backend/static
+decky_loader/static/*
 
 # ignore settings.json
 # prevents leaking login details

.vscode/launch.json

@@ -14,8 +14,8 @@
             "name": "Run (Local)",
             "type": "python",
             "request": "launch",
-            "program": "${workspaceFolder}/backend/main.py",
-            "cwd": "${workspaceFolder}/backend",
+            "module": "decky_loader",
+            "cwd": "${workspaceFolder}",
             "console": "integratedTerminal",
             "env": {
                 "PLUGIN_PATH": "${workspaceFolder}/plugins"

.vscode/tasks.json

@@ -105,7 +105,7 @@
             "detail": "Deploy dev PluginLoader to deck",
             "type": "shell",
             "group": "none",
-            "command": "rsync -azp --delete --rsh='ssh -p ${config:deckport} ${config:deckkey}' --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='requirements.txt' --exclude='backend/__pycache__/' --exclude='.gitignore' . deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader",
+            "command": "rsync -azp --delete --rsh='ssh -p ${config:deckport} ${config:deckkey}' --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='requirements.txt' --exclude='decky_loader/__pycache__/' --exclude='.gitignore' . deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader",
             "problemMatcher": []
         },
         // RUN
@@ -117,15 +117,15 @@
             "dependsOn": [
                 "checkforsettings"
             ],
-            "command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'export PLUGIN_PATH=${config:deckdir}/homebrew/dev/plugins; export CHOWN_PLUGIN_PATH=0; export LOG_LEVEL=DEBUG; cd ${config:deckdir}/homebrew/services; echo '${config:deckpass}' | sudo -SE python3 ${config:deckdir}/homebrew/dev/pluginloader/backend/main.py'",
+            "command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'export PLUGIN_PATH=${config:deckdir}/homebrew/dev/plugins; export CHOWN_PLUGIN_PATH=0; export LOG_LEVEL=DEBUG; export UNPRIVILEGED_PATH=${config:deckdir}/homebrew/services; cd ${config:deckdir}/homebrew/dev/pluginloader; echo '${config:deckpass}' | sudo -SE python3 -m decky_loader'",
             "problemMatcher": []
         },
         {
             "label": "runpylocal",
             "detail": "Run PluginLoader from python locally",
             "type": "shell",
             "group": "none",
-            "command": "export PLUGIN_PATH=${workspaceFolder}/plugins; export CHOWN_PLUGIN_PATH=0; sudo -E python3 ${workspaceFolder}/backend/main.py",
+            "command": "export PLUGIN_PATH=${workspaceFolder}/plugins; export CHOWN_PLUGIN_PATH=0; cd ${workspaceFolder}; sudo -E python3 -m decky_loader",
             "problemMatcher": []
         },
         // ALL-IN-ONES

contrib/deck.sh

@@ -303,7 +303,7 @@ printf "Copying relevant files to install directory\n\n"
 ssh deck@${DECKIP} -p ${SSHPORT} ${IDENINVOC} "mkdir -p $INSTALLDIR/pluginloader && mkdir -p $INSTALLDIR/plugins" &> '/dev/null'
 
 ### copy files for PluginLoader
-rsync -avzp --rsh="ssh -p $SSHPORT $IDENINVOC" --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='requirements.txt' --exclude='backend/__pycache__/' --exclude='.gitignore' --delete ${CLONEDIR}/pluginloader/* deck@${DECKIP}:${INSTALLDIR}/pluginloader &> '/dev/null'
+rsync -avzp --rsh="ssh -p $SSHPORT $IDENINVOC" --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='requirements.txt' --exclude='decky_loader/__pycache__/' --exclude='.gitignore' --delete ${CLONEDIR}/pluginloader/* deck@${DECKIP}:${INSTALLDIR}/pluginloader &> '/dev/null'
 
 if ! [[ $? -eq 0 ]]; then
     printf "Error occurred when copying $CLONEDIR/pluginloader/ to $INSTALLDIR/pluginloader/\n"
@@ -321,10 +321,10 @@ fi
 ## TODO: direct contributors to wiki for this info?
 
 printf "Run these commands to deploy your local changes to the deck:\n"
-printf "'rsync -avzp --mkpath --rsh=""\"ssh -p $SSHPORT $IDENINVOC\""" --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='requirements.txt' --exclude='backend/__pycache__/' --exclude='.gitignore' --delete $CLONEDIR/pluginloader/* deck@$DECKIP:$INSTALLDIR/pluginloader/'\n"
+printf "'rsync -avzp --mkpath --rsh=""\"ssh -p $SSHPORT $IDENINVOC\""" --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='requirements.txt' --exclude='decky_loader/__pycache__/' --exclude='.gitignore' --delete $CLONEDIR/pluginloader/* deck@$DECKIP:$INSTALLDIR/pluginloader/'\n"
 printf "'rsync -avzp --mkpath --rsh=""\"ssh -p $SSHPORT $IDENINVOC\""" --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='node_modules/' --exclude='src/' --exclude='*.log' --exclude='.gitignore' --exclude='package-lock.json' --delete $CLONEDIR/pluginname deck@$DECKIP:$INSTALLDIR/plugins'\n\n"
 
-printf "Run in console or in a script this command to run your development version:\n'ssh deck@$DECKIP -p $SSHPORT $IDENINVOC 'export PLUGIN_PATH=$INSTALLDIR/plugins; export CHOWN_PLUGIN_PATH=0; echo 'steam' | sudo -SE python3 $INSTALLDIR/pluginloader/backend/main.py'\n"
+printf "Run in console or in a script this command to run your development version:\n'ssh deck@$DECKIP -p $SSHPORT $IDENINVOC 'export PLUGIN_PATH=$INSTALLDIR/plugins; export CHOWN_PLUGIN_PATH=0; cd $INSTALLDIR/pluginloader; echo 'steam' | sudo -SE python3 -m decky_loader'\n"
 
 ## Disable Releases versions if they exist
 

contrib/nodeck.sh

@@ -163,6 +163,6 @@ pnpmtransbundle ${CLONEDIR}/plugintemplate "template"
 
 printf "Plugin Loader is located at '${CLONEDIR}/pluginloader/'.\n"
 
-printf "Run in console or in a script these commands to run your development version:\n'export PLUGIN_PATH=${CLONEDIR}/plugins; export CHOWN_PLUGIN_PATH=0; sudo -E python3 ${CLONEDIR}/pluginloader/backend/main.py'\n"
+printf "Run in console or in a script these commands to run your development version:\n'export PLUGIN_PATH=${CLONEDIR}/plugins; export CHOWN_PLUGIN_PATH=0; cd ${CLONEDIR}/pluginloader; sudo -E python3 -m decky_loader'\n"
 
 printf "All done!\n"

decky_loader.spec

@@ -0,0 +1,32 @@
+# -*- mode: python ; coding: utf-8 -*-
+import os
+from PyInstaller.building.build_main import Analysis
+from PyInstaller.building.api import EXE, PYZ
+from PyInstaller.utils.hooks import copy_metadata
+
+a = Analysis(
+    ['pyi-main.py'],
+    datas=[
+        ('decky_loader/static', 'decky_loader/static'),
+        ('decky_loader/legacy', 'decky_loader/legacy'),
+        ('decky_loader/locales', 'decky_loader/locales'),
+    ] + copy_metadata('decky_loader'),
+    hiddenimports=['sqlite3'],
+)
+pyz = PYZ(a.pure, a.zipped_data)
+
+noconsole = bool(os.getenv('DECKY_NOCONSOLE'))
+name = "PluginLoader"
+if noconsole:
+    name += "_noconsole"
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    name=name,
+    upx=True,
+    console=not noconsole,
+)

decky_loader/__main__.py

@@ -0,0 +1,49 @@
+import sys
+from .localplatform import chmod
+
+# Change PyInstaller files permissions
+if hasattr(sys, '_MEIPASS'):
+    chmod(sys._MEIPASS, 755)
+
+from asyncio import new_event_loop, set_event_loop
+from logging import basicConfig, getLogger
+import multiprocessing
+from os import path
+import sys
+
+from .helpers import get_effective_user_id, get_system_pythonpaths
+from .localplatform import ON_WINDOWS, get_log_level
+from .main import PluginManager
+
+logger = getLogger("Main")
+
+
+def main():
+    basicConfig(
+        level=get_log_level(),
+        format="[%(module)s][%(levelname)s]: %(message)s"
+    )
+
+    if ON_WINDOWS:
+        # Fix windows/flask not recognising that .js means 'application/javascript'
+        import mimetypes
+        mimetypes.add_type('application/javascript', '.js')
+
+        # Required for multiprocessing support in frozen files
+        multiprocessing.freeze_support()
+    else:
+      if get_effective_user_id() != 0:
+        logger.warning(f"decky is running as an unprivileged user, this is not officially supported and may cause issues")
+
+    # Append the loader's plugin path to the recognized python paths
+    sys.path.append(path.join(path.dirname(__file__), "plugin_api"))
+
+    # Append the system and user python paths
+    sys.path.extend(get_system_pythonpaths())
+
+    loop = new_event_loop()
+    set_event_loop(loop)
+    PluginManager(loop).run()
+
+if __name__ == '__main__':
+    main()

backend/browser.py → decky_loader/browser.py

@@ -1,24 +1,21 @@
 # Full imports
 import json
-# import pprint
-# from pprint import pformat
 
 # Partial imports
-from aiohttp import ClientSession, web
-from asyncio import get_event_loop, sleep
-from concurrent.futures import ProcessPoolExecutor
+from aiohttp import ClientSession
+from asyncio import sleep
 from hashlib import sha256
 from io import BytesIO
 from logging import getLogger
-from os import R_OK, W_OK, path, rename, listdir, access, mkdir
+from os import R_OK, W_OK, path, listdir, access, mkdir
 from shutil import rmtree
 from time import time
 from zipfile import ZipFile
-from localplatform import chown, chmod
 
 # Local modules
-from helpers import get_ssl_context, download_remote_binary_to_path
-from injector import get_gamepadui_tab
+from .helpers import get_ssl_context, download_remote_binary_to_path
+from .injector import get_gamepadui_tab
+from .localplatform import chown, chmod
 
 logger = getLogger("Browser")
 

backend/helpers.py → decky_loader/helpers.py

@@ -1,19 +1,20 @@
+import importlib.metadata
 import re
 import ssl
 import uuid
 import os
-import sys
 import subprocess
 from hashlib import sha256
 from io import BytesIO
 
 import certifi
 from aiohttp.web import Response, middleware
 from aiohttp import ClientSession
-import localplatform
-from customtypes import UserType
 from logging import getLogger
 
+from . import localplatform
+from .customtypes import UserType
+
 REMOTE_DEBUGGER_UNIT = "steam-web-debug-portforward.service"
 
 # global vars
@@ -48,16 +49,9 @@ def mkdir_as_user(path):
 
 # Fetches the version of loader
 def get_loader_version() -> str:
-    # FIXME: this should really come from package metadata
-    env_version = os.getenv('DECKY_VERSION')
-    if env_version:
-        return env_version
-
     try:
-        with open(os.path.join(os.getcwd(), ".loader.version"), "r", encoding="utf-8") as version_file:
-            return version_file.readline().strip()
-    except Exception as e:
-        logger.warn(f"Failed to execute get_loader_version(): {str(e)}")
+        return "v" + importlib.metadata.version('decky_loader')
+    except importlib.metadata.PackageNotFoundError:
         return "unknown"
 
 # returns the appropriate system python paths

backend/injector.py → decky_loader/injector.py

@@ -2,10 +2,9 @@
 
 from asyncio import sleep
 from logging import getLogger
-from traceback import format_exc
 from typing import List
 
-from aiohttp import ClientSession, WSMsgType
+from aiohttp import ClientSession
 from aiohttp.client_exceptions import ClientConnectorError, ClientOSError
 from asyncio.exceptions import TimeoutError
 import uuid

backend/loader.py → decky_loader/loader.py

@@ -2,16 +2,16 @@
 from json.decoder import JSONDecodeError
 from logging import getLogger
 from os import listdir, path
+from os.path import exists
 from pathlib import Path
 from traceback import print_exc
 
 from aiohttp import web
-from os.path import exists
 from watchdog.events import RegexMatchingEventHandler
 from watchdog.observers import Observer
 
-from injector import get_tab, get_gamepadui_tab
-from plugin import PluginWrapper
+from .injector import get_tab, get_gamepadui_tab
+from .plugin import PluginWrapper
 
 class FileChangeHandler(RegexMatchingEventHandler):
     def __init__(self, queue, plugin_path) -> None:
@@ -225,4 +225,4 @@ async def handle_backend_reload_request(self, request):
 
         await self.reload_queue.put((plugin.file, plugin.plugin_directory))
 
-        return web.Response(status=200)
+        return web.Response(status=200)

backend/localplatform.py → decky_loader/localplatform.py

@@ -4,11 +4,11 @@
 ON_LINUX = not ON_WINDOWS
 
 if ON_WINDOWS:
-    from localplatformwin import *
-    import localplatformwin as localplatform
+    from .localplatformwin import *
+    from . import localplatformwin as localplatform
 else:
-    from localplatformlinux import *
-    import localplatformlinux as localplatform
+    from .localplatformlinux import *
+    from . import localplatformlinux as localplatform
 
 def get_privileged_path() -> str:
     '''Get path accessible by elevated user. Holds plugins, decky loader and decky loader configs'''