langchain[patch]: update metadata and default expand param for conflu…

…ence loader (#4220) * update metadata and default expand param for confluence loader * format * Fix test * Format --------- Co-authored-by: Chris Chen <[email protected]> Co-authored-by: jacoblee93 <[email protected]>
langchain-ai · Feb 7, 2024 · 12204d6 · 12204d6
1 parent d4c5dd5
commit 12204d6
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 8 deletions.
diff --git a/langchain/src/document_loaders/tests/confluence.test.ts b/langchain/src/document_loaders/tests/confluence.test.ts
@@ -36,11 +36,15 @@ test("Test ConfluenceLoader and fetchConfluenceData calls", async () => {
   // until the size of the response is 0
   const fetchConfluenceDataMock = jest
     .spyOn(loader, "fetchConfluenceData")
-    .mockImplementationOnce(() =>
-      Promise.resolve({ size: 2, results: fakeResponse })
+    .mockImplementationOnce(
+      () =>
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Promise.resolve({ size: 2, results: fakeResponse }) as any
     )
-    .mockImplementationOnce(() =>
-      Promise.resolve({ size: 2, results: fakeResponse })
+    .mockImplementationOnce(
+      () =>
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Promise.resolve({ size: 2, results: fakeResponse }) as any
     )
     .mockImplementationOnce(() => Promise.resolve({ size: 0, results: [] }));
 
@@ -57,15 +61,15 @@ test("Test ConfluenceLoader and fetchConfluenceData calls", async () => {
   // Ensure the arguments are correct for each call
   expect(fetchConfluenceDataMock).toHaveBeenNthCalledWith(
     1,
-    "https://example.atlassian.net/wiki/rest/api/content?spaceKey=SPACEKEY&limit=25&start=0&expand=body.storage"
+    "https://example.atlassian.net/wiki/rest/api/content?spaceKey=SPACEKEY&limit=25&start=0&expand=body.storage,version"
   );
   expect(fetchConfluenceDataMock).toHaveBeenNthCalledWith(
     2,
-    "https://example.atlassian.net/wiki/rest/api/content?spaceKey=SPACEKEY&limit=25&start=2&expand=body.storage"
+    "https://example.atlassian.net/wiki/rest/api/content?spaceKey=SPACEKEY&limit=25&start=2&expand=body.storage,version"
   );
   expect(fetchConfluenceDataMock).toHaveBeenNthCalledWith(
     3,
-    "https://example.atlassian.net/wiki/rest/api/content?spaceKey=SPACEKEY&limit=25&start=4&expand=body.storage"
+    "https://example.atlassian.net/wiki/rest/api/content?spaceKey=SPACEKEY&limit=25&start=4&expand=body.storage,version"
   );
 
   // Check if the generated URLs in the metadata are correct

diff --git a/langchain/src/document_loaders/web/confluence.ts b/langchain/src/document_loaders/web/confluence.ts
@@ -22,11 +22,20 @@ export interface ConfluencePagesLoaderParams {
 export interface ConfluencePage {
   id: string;
   title: string;
+  type: string;
   body: {
     storage: {
       value: string;
     };
   };
+  status: string;
+  version?: {
+    number: number;
+    when: string;
+    by: {
+      displayName: string;
+    };
+  };
 }
 
 /**
@@ -76,7 +85,7 @@ export class ConfluencePagesLoader extends BaseDocumentLoader {
     username,
     accessToken,
     limit = 25,
-    expand = "body.storage",
+    expand = "body.storage,version",
     personalAccessToken,
   }: ConfluencePagesLoaderParams) {
     super();
@@ -212,8 +221,14 @@ export class ConfluencePagesLoader extends BaseDocumentLoader {
     return new Document({
       pageContent: textWithoutEmptyLines,
       metadata: {
+        id: page.id,
+        status: page.status,
         title: page.title,
+        type: page.type,
         url: pageUrl,
+        version: page.version?.number,
+        updated_by: page.version?.by?.displayName,
+        updated_at: page.version?.when,
       },
     });
   }