Merge branch 'main' into bug/17352_gracefully_handle_malformed_responses

llama-index-integrations/readers/llama-index-readers-web/llama_index/readers/web/rss/base.py

@@ -40,18 +40,27 @@ def load_data(self, urls: List[str]) -> List[Document]:
         for url in urls:
             parsed = feedparser.parse(url)
             for entry in parsed.entries:
-                doc_id = entry.id or entry.link
-                if "content" in entry:
-                    data = entry.content[0].value
-                else:
-                    data = entry.description or entry.summary
+                doc_id = getattr(entry, "id", None) or getattr(entry, "link", None)
+                data = entry.get("content", [{}])[0].get(
+                    "value", entry.get("description", entry.get("summary", ""))
+                )
 
                 if self.html_to_text:
                     import html2text
 
                     data = html2text.html2text(data)
 
-                extra_info = {"title": entry.title, "link": entry.link}
-                documents.append(Document(text=data, id_=doc_id, extra_info=extra_info))
+                extra_info = {
+                    "title": getattr(entry, "title", None),
+                    "link": getattr(entry, "link", None),
+                    "date": getattr(entry, "published", None),
+                }
+
+                if doc_id:
+                    documents.append(
+                        Document(text=data, id_=doc_id, extra_info=extra_info)
+                    )
+                else:
+                    documents.append(Document(text=data, extra_info=extra_info))
 
         return documents

llama-index-integrations/readers/llama-index-readers-web/pyproject.toml

@@ -45,7 +45,7 @@ license = "MIT"
 maintainers = ["HawkClaws", "Hironsan", "NA", "an-bluecat", "bborn", "jasonwcfan", "kravetsmic", "pandazki", "ruze00", "selamanse", "thejessezhang"]
 name = "llama-index-readers-web"
 readme = "README.md"
-version = "0.3.1"
+version = "0.3.2"
 
 [tool.poetry.dependencies]
 python = ">=3.9,<4.0"