Add context to model requests for MacOS using a11y APIs (#98)

lib/a11yHelpers.py

@@ -0,0 +1,48 @@
+# Adapted from MIT licensed code here:
+# https://github.com/phillco/talon-axkit/blob/main/dictation/dictation_context.py
+
+from talon import Context, Module, ui
+from talon.types import Span
+
+ctx = Context()
+ctx.matches = r"""
+os: mac
+app: code
+"""
+
+mod = Module()
+
+
+@mod.action_class
+class GenericActions:
+    def a11y_get_context_of_editor(selection: str) -> str:
+        """Creates a `AccessibilityContext` representing the state of the document"""
+        # If we aren't in a valid editor on a valid platform, just return an empty string
+        return ""
+
+
+@ctx.action_class("user")
+class Actions:
+
+    def a11y_get_context_of_editor(selection: str) -> str:
+        """Creates a `AccessibilityContext` representing the state of the document"""
+
+        el = ui.focused_element()
+
+        if not el or not el.attrs:
+            raise ValueError("No valid a11y element")
+
+        # Only return extra context if we are in an editor
+        if not el.get("AXRoleDescription") == "editor":
+            return ""
+
+        context = el.get("AXValue")
+
+        if context is None:
+            raise ValueError("No accessibility information present")
+        # This probably means that a11y support is not enabled (we can't get more than just the current
+        # selection)or that we selected the entire document
+        if context == selection:
+            return ""
+
+        return context

lib/modelHelpers.py

@@ -31,6 +31,17 @@ def get_token() -> str:
         raise Exception(message)
 
 
+def make_prompt_from_editor_ctx(ctx: str):
+    """Add the editor context to the prompt"""
+    if not ctx:
+        return ""
+
+    return (
+        "\n The user is inside a code editor. Use the content of the editor to improve the response and make it tailored to the specific context. The content is as follows: \n\n\n"
+        + ctx
+    )
+
+
 def generate_payload(
     prompt: str, content: str, tools: Optional[list[Tool]] = None
 ) -> Tuple[Headers, Data]:
@@ -46,7 +57,7 @@ def generate_payload(
         f"\nThe user is currently in a code editor for {language}."
         if language != ""
         else ""
-    )
+    ) + make_prompt_from_editor_ctx(actions.user.a11y_get_context_of_editor(content))
 
     headers = {
         "Content-Type": "application/json",