remove folder with only meta files

embeddings-benchmark · Aug 14, 2024 · 678e117 · 678e117
1 parent 1e028d6
commit 678e117
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 2 deletions.
diff --git a/results/e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/model_meta.json b/results/e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/model_meta.json
diff --git a/...s/multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/model_meta.json b/...s/multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/model_meta.json
diff --git a/scripts/normalize_results_folder.py b/scripts/normalize_results_folder.py
@@ -68,6 +68,17 @@ def resolve_conflict(current_path: Path, expected_path: Path) -> None:
         resolve_conflict_result(current_path, expected_path)
 
 
+def remove_folders_with_only_meta_files(results_folder: Path) -> None:
+    """Remove folders that only contain a meta file."""
+    for folder in results_folder.glob("*/*"):
+        if len(list(folder.glob("*.json"))) == 1:
+            meta_file = list(folder.glob("*.json"))[0]
+            if meta_file.name == "model_meta.json":
+                logger.info(f"Removing folder {folder}")
+                meta_file.unlink()
+                folder.rmdir()
+
+
 def main(attempt_to_resolve_conflict: bool) -> None:
     """Main function."""
     results_folder = Path(__file__).parent.parent / "results"
@@ -107,6 +118,8 @@ def main(attempt_to_resolve_conflict: bool) -> None:
     if conflict_encountered and not attempt_to_resolve_conflict:
         raise Exception("Conflicts encountered.")
 
+    remove_folders_with_only_meta_files(results_folder)
+
 
 if __name__ == "__main__":
     logging.basicConfig(