Post Process Paragraphs to Improve Chunking (#385)

* post processed paragraphs to check for single line paragraphs and merge them

* add clarifying commetns

src/lib/vectordb.ts

@@ -23,9 +23,29 @@ async function addReport(url: string, markdown: string) {
     .map((p) => p.trim())
     .filter((p) => p.length > 0)
 
+  let prefix = ''
+  const mergedParagraphs: string[] = []
+
+  // Combine standalone headers (titles without body) with the next paragraph that has a body.
+  for (let i = 0; i < paragraphs.length; i++) {
+    const current = paragraphs[i]
+    const hasBody = current.split('\n').length > 1
+
+    if (!hasBody) {
+      prefix += (prefix ? '\n' : '') + current
+    } else {
+      mergedParagraphs.push((prefix ? prefix + '\n' : '') + current)
+      prefix = ''
+    }
+  }
+
+  if (prefix) {
+    mergedParagraphs.push(prefix)
+  }
+
   const documentChunks: { chunk: string; paragraph: string }[] = []
 
-  paragraphs.forEach((paragraph) => {
+  mergedParagraphs.forEach((paragraph) => {
     for (let i = 0; i < paragraph.length; i += CHUNK_SIZE - overlapSize) {
       const chunk = paragraph.slice(i, i + CHUNK_SIZE).trim()
       if (chunk.length > 0) {

src/prompts/followUp/scope12.ts

@@ -43,7 +43,7 @@ NEVER CALCULATE ANY EMISSIONS. ONLY REPORT THE DATA AS IT IS IN THE PDF. If you
 Example - feel free to add more fields and relevant data:
 {
   "scope12": [{
-    "year": 2021,
+    "year": 2023,
     "scope1": {
       "total": 12.3
     },