Fallback to json5 loader if json.loads cannot parse complex json str

JSON5 spec is more flexible, try to load using a fast json5 parser if
the stricter json.loads from the standard library can't load the
raw complex json string into a python dictionary/list

pyproject.toml

@@ -88,6 +88,7 @@ dependencies = [
     "anthropic == 0.26.1",
     "docx2txt == 0.8",
     "google-generativeai == 0.8.3",
+    "pyjson5 == 1.6.7",
 ]
 dynamic = ["version"]
 

src/khoj/processor/conversation/utils.py

@@ -15,6 +15,7 @@
 from typing import Any, Callable, Dict, List, Optional
 
 import PIL.Image
+import pyjson5
 import requests
 import tiktoken
 import yaml
@@ -566,11 +567,11 @@ def replace_unescaped_quotes(match):
 
     # See which json loader can load the processed JSON as valid
     errors = ""
-    json_loaders_to_try = [json.loads]
+    json_loaders_to_try = [json.loads, pyjson5.loads]
     for loads in json_loaders_to_try:
         try:
             return loads(processed)
-        except json.JSONDecodeError as e:
+        except [json.JSONDecodeError, pyjson5.Json5Exception] as e:
             errors += f"\n\n{e}"
 
     # If all loaders fail, raise the aggregated error