diff --git a/crates/tabby-scheduler/src/repository.rs b/crates/tabby-scheduler/src/repository.rs
index ce4a4472df16..fb81886099b0 100644
--- a/crates/tabby-scheduler/src/repository.rs
+++ b/crates/tabby-scheduler/src/repository.rs
@@ -1,7 +1,11 @@
-use std::process::Command;
+use std::{collections::HashSet, fs, process::Command};
 
 use anyhow::{anyhow, Result};
-use tabby_common::config::{Config, RepositoryConfig};
+use tabby_common::{
+    config::{Config, RepositoryConfig},
+    path::repositories_dir,
+};
+use tracing::warn;
 
 trait ConfigExt {
     fn sync_repositories(&self) -> Result<()>;
@@ -9,7 +13,9 @@ trait ConfigExt {
 
 impl ConfigExt for Config {
     fn sync_repositories(&self) -> Result<()> {
+        let mut names = HashSet::new();
         for repository in self.repositories.iter() {
+            names.insert(repository.name());
             if repository.is_local_dir() {
                 if !repository.dir().exists() {
                     panic!("Directory {} does not exist", repository.dir().display());
@@ -19,6 +25,22 @@ impl ConfigExt for Config {
             }
         }
 
+        for file in fs::read_dir(repositories_dir())?.filter_map(Result::ok) {
+            let metadata = file.metadata()?;
+            let filename = file.file_name();
+            if metadata.is_file() {
+                warn!("An unrelated file {:?} was found in repositories directory, It will now be removed...", filename);
+                // There shouldn't be any files under repositories dir.
+                fs::remove_file(file.path())?;
+            } else if metadata.is_dir() {
+                let filename = filename.to_str().ok_or(anyhow!("Invalid file name"))?;
+                if !names.contains(filename) {
+                    warn!("An unrelated directory {:?} was found in repositories directory, It will now be removed...", filename);
+                    fs::remove_dir_all(file.path())?;
+                }
+            }
+        }
+
         Ok(())
     }
 }
diff --git a/website/docs/configuration.md b/website/docs/configuration.md
index 54a95a163b34..c79ac960ec9b 100644
--- a/website/docs/configuration.md
+++ b/website/docs/configuration.md
@@ -11,17 +11,20 @@ Tabby server will look for a configuration file at `~/.tabby/config.toml` for ad
 To enable repository level context for code completion, you can add the following to your configuration file:
 
 ```toml title="~/.tabby/config.toml"
-# Index two repositories' source code as additional context for code completion.
+# Index three repositories' source code as additional context for code completion.
 
 [[repositories]]
+name = "tabby"
 git_url = "https://github.com/TabbyML/tabby.git"
 
 # git through ssh protocol.
 [[repositories]]
+name = "CTranslate2"
 git_url = "git@github.com:OpenNMT/CTranslate2.git"
 
 # local directory is also supported!
 [[repositories]]
+name = "repository_a"
 git_url = "file:///home/users/repository_a"
 ```