feat: try using readability to parse web pages

pyproject.toml

@@ -3,7 +3,7 @@ name = "reasonify-headless"
 description = "headless implementation for the Reasonify Agent"
 license = { text = "MIT" }
 dynamic = [ "version" ]
-requires-python = ">=3.12"
+requires-python = "~=3.12"
 authors = [ { name = "Muspi Merol", email = "[email protected]" } ]
 readme = "README.md"
 dependencies = [
@@ -16,6 +16,7 @@ dependencies = [
   "html-text~=0.6.2",
   "html2text2~=1.0.0",
   "isomorphic-fetch~=0.0.0.0.dev2",
+  "python-readability~=0.0.2",
 ]
 
 [build-system]

reasonify-headless/reasonify/tools/fetch.py

@@ -32,4 +32,8 @@ async def _fetch(url: str, strategy: Strategy):
     except OSError:
         res = await fetch(f"/api/proxy?url={url}")
 
-    return res.status, post_process(pre_process(res.text, strategy))
+    from readability import parse
+
+    text = parse(res.text, keep_classes=True).content or res.text
+
+    return res.status, post_process(pre_process(text, strategy))