Packaging rework (#531)

* fix: get rid of title view jank on latest beta

* Count the number of installs for each plugin (#557)

* Bump aiohttp from 3.8.4 to 3.8.5 in /backend (#558)

* fix: include Decky version in request for index.js

This avoids the If-Modified-Since logic in aiohttp and ensures Steam doesn't cache old JS,
even if the timestamps are normalized.

* fix: clean up shellcheck warnings in act runner script

* fix: gitignore settings/

* fix: ensure state directories exist when running without the installer

* feat: determine root directory correctly when running from in-tree

* fix: fix typo in CI script

* refactor: build a proper Python package with poetry

* refactor: move decky_plugin under the poetry structure

There's no need to special case it anymore, just treat it like any other Python module.

* sandboxed_plugin: better fix, attempt 2

---------

Co-authored-by: AAGaming <[email protected]>
Co-authored-by: Party Wumpus <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/build-win.yml

@@ -25,13 +25,15 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "3.11.4"
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: false
 
     - name: Install Python dependencies ⬇️
       working-directory: ./backend
-      run: |
-        python -m pip install --upgrade pip
-        pip install pyinstaller==5.13.0
-        pip install -r requirements.txt
+      run: C:\Users\runneradmin\.local\bin\poetry install --no-interaction
 
     - name: Install JS dependencies ⬇️
       working-directory: ./frontend
@@ -44,16 +46,18 @@ jobs:
       run: pnpm run build
 
     - name: Build Python Backend 🛠️
-      run: pyinstaller --noconfirm --onefile --name "PluginLoader" --add-data "./backend/static;/static" --add-data "./backend/locales;/locales" --add-data "./plugin;/plugin" --hidden-import=logging.handlers --hidden-import=sqlite3 ./backend/main.py
+      working-directory: ./backend
+      run: C:\Users\runneradmin\.local\bin\poetry run pyinstaller pyinstaller.spec
 
     - name: Build Python Backend (noconsole) 🛠️
-      run: pyinstaller --noconfirm --noconsole --onefile --name "PluginLoader_noconsole" --add-data "./backend/static;/static" --add-data "./backend/locales;/locales" --add-data "./plugin;/plugin" --hidden-import=logging.handlers --hidden-import=sqlite3 ./backend/main.py
-
+      working-directory: ./backend
+      run: $env:DECKY_NOCONSOLE = 1; C:\Users\runneradmin\.local\bin\poetry run pyinstaller pyinstaller.spec
+
     - name: Upload package artifact ⬆️
       uses: actions/upload-artifact@v3
       with:
         name: PluginLoader Win
         path: |
-          ./dist/PluginLoader.exe
-          ./dist/PluginLoader_noconsole.exe
+          ./backend/dist/PluginLoader.exe
+          ./backend/dist/PluginLoader_noconsole.exe
     

.github/workflows/build.yml

@@ -69,12 +69,14 @@ jobs:
         sudo cp /usr/lib/libsqlite3.so.0.8.6 /usr/lib/x86_64-linux-gnu/ &&
         rm -r /tmp/sqlite-autoconf-3420000
 
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: false
+
     - name: Install Python dependencies ⬇️
       working-directory: ./backend
-      run: |
-        python -m pip install --upgrade pip
-        pip install pyinstaller==5.13.0
-        pip install -r requirements.txt
+      run: poetry install --no-interaction
 
     - name: Install JS dependencies ⬇️
       working-directory: ./frontend
@@ -87,20 +89,21 @@ jobs:
       run: pnpm run build
 
     - name: Build Python Backend 🛠️
-      run: pyinstaller --noconfirm --onefile --name "PluginLoader" --add-data ./backend/static:/static --add-data ./backend/locales:/locales --add-data ./plugin:/plugin --hidden-import=logging.handlers --hidden-import=sqlite3 ./backend/main.py
-
+      working-directory: ./backend
+      run: pyinstaller pyinstaller.spec
+
     - name: Upload package artifact ⬆️
       if: ${{ !env.ACT }}
       uses: actions/upload-artifact@v3
       with:
         name: PluginLoader
-        path: ./dist/PluginLoader
+        path: ./backend/dist/PluginLoader
 
     - name: Download package artifact locally
       if: ${{ env.ACT }}
       uses: actions/upload-artifact@v3
       with:
-        path: ./dist/PluginLoader
+        path: ./backend/dist/PluginLoader
 
   release:
     name: Release stable version of the package
@@ -177,7 +180,7 @@ jobs:
       with:
         name: Release ${{ steps.ready_tag.outputs.tag_name }}
         tag_name: ${{ steps.ready_tag.outputs.tag_name }}
-        files: ./dist/PluginLoader
+        files: ./backend/dist/PluginLoader
         prerelease: false
         generate_release_notes: true
 
@@ -264,6 +267,6 @@ jobs:
       with:
         name: Prerelease ${{ steps.ready_tag.outputs.tag_name }}
         tag_name: ${{ steps.ready_tag.outputs.tag_name }}
-        files: ./dist/PluginLoader
+        files: ./backend/dist/PluginLoader
         prerelease: true
         generate_release_notes: true

.github/workflows/edit-check.yml

@@ -22,18 +22,18 @@ jobs:
       with:
         separator: ","
         files: |
-          plugin/*
+          backend/decky_plugin.pyi
           
     - name: Is stub changed
       id: changed-stub
       run: |
         STUB_CHANGED="false"
-        PATHS=(plugin plugin/decky_plugin.pyi)
+        PATHS=(backend backend/decky_plugin.pyi)
         SHA=${{ github.sha }}
         SHA_PREV=HEAD^
         FILES=$(git diff $SHA_PREV..$SHA --name-only -- ${PATHS[@]} | jq -Rsc 'split("\n")[:-1] | join (",")')
-        if [[ "$FILES" == *"plugin/decky_plugin.pyi"* ]]; then
-          $STUB_CHANGED="true"
+        if [[ "$FILES" == *"backend/decky_plugin.pyi"* ]]; then
+          STUB_CHANGED="true"
           echo "Stub has changed, pushing updated stub"
         else
           echo "Stub has not changed, exiting."
@@ -48,7 +48,7 @@ jobs:
       env:
         API_TOKEN_GITHUB: ${{ secrets.GITHUB_TOKEN }}
       with:
-        source_file: 'plugin/decky_plugin.pyi'
+        source_file: 'backend/decky_plugin.pyi'
         destination_repo: 'SteamDeckHomebrew/decky-plugin-template'
         user_email: '[email protected]'
         user_name: 'TrainDoctor'

.github/workflows/typecheck.yml

@@ -12,11 +12,19 @@ jobs:
     steps:
       - uses: actions/checkout@v3 # Check out the repository first.
 
-      - name: Install Python dependencies
+      - name: Set up Python 3.10.6 🐍
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10.6"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
+
+      - name: Install Python dependencies ⬇️
         working-directory: backend
-        run: |
-          python -m pip install --upgrade pip
-          [ -f requirements.txt ] && pip install -r requirements.txt
+        run: poetry install --no-interaction
 
       - name: Install TypeScript dependencies
         working-directory: frontend

.gitignore

@@ -29,7 +29,7 @@ MANIFEST
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
-*.spec
+backend/dist/
 
 # Installer logs
 pip-log.txt
@@ -163,3 +163,4 @@ plugins/*
 act/.directory
 act/artifacts/*
 bin/act
+settings/

.vscode/tasks.json

@@ -38,7 +38,7 @@
             "type": "shell",
             "group": "none",
             "detail": "Check for local runs, create a plugins folder",
-            "command": "rsync -azp --rsh='ssh -p ${config:deckport} ${config:deckkey}' requirements.txt deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader/requirements.txt && ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'python -m ensurepip && python -m pip install --upgrade pip && python -m pip install --upgrade setuptools && python -m pip install -r ${config:deckdir}/homebrew/dev/pluginloader/requirements.txt'",
+            "command": "rsync -azp --rsh='ssh -p ${config:deckport} ${config:deckkey}' pyproject.toml poetry.lock deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader && ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'python -m ensurepip && python -m pip install --upgrade poetry && cd ${config:deckdir}/homebrew/dev/pluginloader/backend && python -m poetry install'",
             "problemMatcher": []
         },
         {
@@ -105,7 +105,7 @@
             "detail": "Deploy dev PluginLoader to deck",
             "type": "shell",
             "group": "none",
-            "command": "rsync -azp --delete --rsh='ssh -p ${config:deckport} ${config:deckkey}' --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='requirements.txt' --exclude='backend/__pycache__/' --exclude='.gitignore' . deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader",
+            "command": "rsync -azp --delete --rsh='ssh -p ${config:deckport} ${config:deckkey}' --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='backend/decky_loader/__pycache__/' --exclude='.gitignore' . deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader",
             "problemMatcher": []
         },
         // RUN

act/run-act.sh

@@ -1,26 +1,29 @@
-#!/bin/bash
+#!/usr/bin/env bash
+set -eo pipefail
 
 type=$1
 # bump=$2
 
 oldartifactsdir="old"
 
-parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
-cd "$parent_path"
-
-artifactfolders=$(find artifacts/ -maxdepth 1 -mindepth 1 -type d)
-if [[ ${#artifactfolders[@]} > 0 ]]; then
-    for i in ${artifactfolders[@]}; do
-        foldername=$(dirname $i)
-        subfoldername=$(basename $i)
-        out=$foldername/$oldartifactsdir/$subfoldername-$(date +'%s')
-        if [[ ! "$subfoldername" =~ "$oldartifactsdir" ]]; then
-            mkdir -p $out
-            mv $i $out
-            printf "Moved "${foldername}"/"${subfoldername}" to "${out}" \n"
-        fi
-    done
-fi
+parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit ; pwd -P )
+cd "$parent_path" || exit
+
+for i in artifacts/*; do
+    if [[ ! -d "$i" ]]; then
+        continue;
+    fi
+    subfoldername=$(basename "$i")
+
+    if [[ "$subfoldername" == "$oldartifactsdir" ]]; then
+        continue;
+    fi
+
+    out=artifacts/$oldartifactsdir/$subfoldername-$(date +'%s')
+    mkdir -p "$out"
+    mv "$i" "$out"
+    echo "Moved artifacts/${subfoldername} to ${out}"
+done
 
 cd ..
 
@@ -35,10 +38,10 @@ else
     printf "Options: 'release' or 'prerelease'\n"
 fi
 
-cd act/artifacts
+cd act/artifacts || exit
 
 if [[ -d "1" ]]; then
-    cd "1/artifact"
+    cd "1/artifact" || exit
     cp "PluginLoader.gz__" "PluginLoader.gz"
     gzip -d "PluginLoader.gz"
     chmod +x PluginLoader

backend/src/browser.py → backend/decky_loader/browser.py

@@ -186,6 +186,18 @@ async def _install(self, artifact: str, name: str, version: str, hash: str):
                 else:
                     logger.fatal(f"Could not fetch from URL. {await res.text()}")
 
+            storeUrl = ""
+            match self.settings.getSetting("store", 0):
+                case 0: storeUrl = "https://plugins.deckbrew.xyz/plugins" # default
+                case 1: storeUrl = "https://testing.deckbrew.xyz/plugins" # testing
+                case 2: storeUrl = self.settings.getSetting("store-url", "https://plugins.deckbrew.xyz/plugins")  # custom
+                case _: storeUrl = "https://plugins.deckbrew.xyz/plugins"
+            logger.info(f"Incrementing installs for {name} from URL {storeUrl} (version {version})")
+            async with ClientSession() as client:
+                res = await client.post(storeUrl+f"/{name}/versions/{version}/increment?isUpdate={isInstalled}", ssl=get_ssl_context())
+                if res.status != 200:
+                    logger.error(f"Server did not accept install count increment request. code: {res.status}")
+
         # Check to make sure we got the file
         if res_zip is None:
             logger.fatal(f"Could not fetch {artifact}")

backend/src/localplatform/localplatformlinux.py → backend/decky_loader/localplatform/localplatformlinux.py

@@ -140,6 +140,8 @@ def get_privileged_path() -> str:
     if path == None:
         path = get_unprivileged_path()
 
+    os.makedirs(path, exist_ok=True)
+
     return path
 
 def _parent_dir(path : str | None) -> str | None:
@@ -159,8 +161,13 @@ def get_unprivileged_path() -> str:
 
     if path == None:
         logger.debug("Unprivileged path is not properly configured. Making something up!")
-        # Expected path of loader binary is /home/deck/homebrew/service/PluginLoader
-        path = _parent_dir(_parent_dir(os.path.realpath(sys.argv[0])))
+
+        if hasattr(sys, 'frozen'):
+            # Expected path of loader binary is /home/deck/homebrew/service/PluginLoader
+            path = _parent_dir(_parent_dir(os.path.realpath(sys.argv[0])))
+        else:
+            # Expected path of this file is $src_root/backend/src/localplatformlinux.py
+            path = _parent_dir(_parent_dir(_parent_dir(__file__)))
 
         if path != None and not os.path.exists(path):
             path = None
@@ -169,6 +176,8 @@ def get_unprivileged_path() -> str:
         logger.warn("Unprivileged path is not properly configured. Defaulting to /home/deck/homebrew")
         path = "/home/deck/homebrew" # We give up
 
+    os.makedirs(path, exist_ok=True)
+
     return path
 
 

backend/src/localplatform/localplatformwin.py → backend/decky_loader/localplatform/localplatformwin.py

@@ -47,6 +47,8 @@ def get_unprivileged_path() -> str:
     if path == None:
         path = os.getenv("PRIVILEGED_PATH", os.path.join(os.path.expanduser("~"), "homebrew"))
 
+    os.makedirs(path, exist_ok=True)
+
     return path
 
 def get_unprivileged_user() -> str:

backend/src/main.py → backend/decky_loader/main.py

@@ -22,7 +22,7 @@
 
 # local modules
 from .browser import PluginBrowser
-from .helpers import (REMOTE_DEBUGGER_UNIT, csrf_middleware, get_csrf_token,
+from .helpers import (REMOTE_DEBUGGER_UNIT, csrf_middleware, get_csrf_token, get_loader_version,
                      mkdir_as_user, get_system_pythonpaths, get_effective_user_id)
 
 from .injector import get_gamepadui_tab, Tab, close_old_tabs
@@ -160,7 +160,7 @@ async def inject_javascript(self, tab: Tab, first: bool=False, request: Request|
             if first:
                 if await tab.has_global_var("deckyHasLoaded", False):
                     await close_old_tabs()
-            await tab.evaluate_js("try{if (window.deckyHasLoaded){setTimeout(() => location.reload(), 100)}else{window.deckyHasLoaded = true;(async()=>{try{while(!window.SP_REACT){await new Promise(r => setTimeout(r, 10))};await import('http://localhost:1337/frontend/index.js')}catch(e){console.error(e)};})();}}catch(e){console.error(e)}", False, False, False)
+            await tab.evaluate_js("try{if (window.deckyHasLoaded){setTimeout(() => location.reload(), 100)}else{window.deckyHasLoaded = true;(async()=>{try{while(!window.SP_REACT){await new Promise(r => setTimeout(r, 10))};await import('http://localhost:1337/frontend/index.js?v=%s')}catch(e){console.error(e)};})();}}catch(e){console.error(e)}" % (get_loader_version(), ), False, False, False)
         except:
             logger.info("Failed to inject JavaScript into tab\n" + format_exc())
             pass
@@ -180,9 +180,6 @@ def main():
       if get_effective_user_id() != 0:
         logger.warning(f"decky is running as an unprivileged user, this is not officially supported and may cause issues")
 
-    # Append the loader's plugin path to the recognized python paths
-    sys.path.append(path.join(path.dirname(__file__), "..", "plugin"))
-
     # Append the system and user python paths
     sys.path.extend(get_system_pythonpaths())
 

backend/src/plugin/sandboxed_plugin.py → backend/decky_loader/plugin/sandboxed_plugin.py

@@ -74,9 +74,9 @@ def initialize(self, socket: LocalSocket):
             syspath.append(path.join(environ["DECKY_PLUGIN_DIR"], "py_modules"))
 
             #TODO: FIX IN A LESS CURSED WAY
-            keys = [key.replace("src.", "") for key in sysmodules if key.startswith("src.")]
+            keys = [key for key in sysmodules if key.startswith("decky_loader.")]
             for key in keys:
-                sysmodules[key] = sysmodules["src"].__dict__[key]
+                sysmodules[key.replace("decky_loader.", "")] = sysmodules[key]
 
             spec = spec_from_file_location("_", self.file)
             assert spec is not None

backend/main.py

@@ -1,4 +1,4 @@
-# This file is needed to make the relative imports in src/ work properly.
+# This file is needed to make the relative imports in decky_loader/ work properly.
 if __name__ == "__main__":
-    from src.main import main
+    from decky_loader.main import main
     main()