fix(index): delete skipped documents for StructuredDoc

crates/tabby-index/src/structured_doc/public.rs

@@ -14,8 +14,14 @@
 use super::{create_structured_doc_builder, types::BuildStructuredDoc};
 use crate::{indexer::TantivyDocBuilder, Indexer};
 
+/// StructuredDocState is used to track the state of the document source.
+/// It is used to determine whether the document should be updated or deleted.
 pub struct StructuredDocState {
+    // updated_at is the time when the document was last updated.
     pub updated_at: DateTime<Utc>,
+    // deleted indecates whether the document should be deleted in indexer
+    // for example, a closed pull request will be marked as deleted, and
+    // the indexer will remove it from the index.
     pub deleted: bool,
 }
 
@@ -37,7 +43,7 @@
         }
 
         if state.deleted {
             return self.delete(document.id()).await;
         }
 
         stream! {
@@ -53,14 +59,14 @@
         true
     }
 
     pub async fn delete(&self, id: &str) -> bool {
         if self.indexer.is_indexed(id) {
             self.indexer.delete(id);
             true
         } else {
             false
         }
     }
 
     pub fn commit(self) {
         self.indexer.commit();