eval abc

FlagEmbedding/abc/evaluation/data_loader.py

@@ -17,10 +17,10 @@ class AbsEvalDataLoader(ABC):
 
     Args:
         eval_name (str): The experiment name of current evaluation.
-        dataset_dir (str, optional): path to the datasets. Defaults to :data:`None`.
-        cache_dir (str, optional): Path to HuggingFace cache directory. Defaults to :data:`None`.
-        token (str, optional): HF_TOKEN to access the private datasets/models in HF. Defaults to :data:`None`.
-        force_redownload: If True, will force redownload the dataset to cover the local dataset. Defaults to :data:`False`.
+        dataset_dir (str, optional): path to the datasets. Defaults to ``None``.
+        cache_dir (str, optional): Path to HuggingFace cache directory. Defaults to ``None``.
+        token (str, optional): HF_TOKEN to access the private datasets/models in HF. Defaults to ``None``.
+        force_redownload: If True, will force redownload the dataset to cover the local dataset. Defaults to ``False``.
     """
     def __init__(
         self,
@@ -78,7 +78,7 @@ def check_splits(self, splits: Union[str, List[str]], dataset_name: Optional[str
 
         Args:
             splits (Union[str, List[str]]): Splits to check.
-            dataset_name (Optional[str], optional): Name of dataset to check. Defaults to None.
+            dataset_name (Optional[str], optional): Name of dataset to check. Defaults to ``None``.
 
         Returns:
             List[str]: The available splits.
@@ -98,7 +98,7 @@ def load_corpus(self, dataset_name: Optional[str] = None) -> datasets.DatasetDic
         """Load the corpus from the dataset.
 
         Args:
-            dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
 
         Returns:
             datasets.DatasetDict: A dict of corpus with id as key, title and text as value.
@@ -116,8 +116,8 @@ def load_qrels(self, dataset_name: Optional[str] = None, split: str = 'test') ->
         """Load the qrels from the dataset.
 
         Args:
-            dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
-            split (str, optional): The split to load relevance from. Defaults to :data:'test'.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            split (str, optional): The split to load relevance from. Defaults to ``'test'``.
 
         Raises:
             ValueError
@@ -144,8 +144,8 @@ def load_queries(self, dataset_name: Optional[str] = None, split: str = 'test')
         """Load the queries from the dataset.
 
         Args:
-            dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
-            split (str, optional): The split to load queries from. Defaults to :data:`'test'`.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            split (str, optional): The split to load queries from. Defaults to ``'test'``.
 
         Raises:
             ValueError
@@ -176,8 +176,8 @@ def _load_remote_corpus(
         """Abstract method to load corpus from remote dataset, to be overrode in child class.
 
         Args:
-            dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
-            save_dir (Optional[str], optional): Path to save the new downloaded corpus. Defaults to :data:`None`.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            save_dir (Optional[str], optional): Path to save the new downloaded corpus. Defaults to ``None``.
 
         Raises:
             NotImplementedError: Loading remote corpus is not implemented.
@@ -196,9 +196,9 @@ def _load_remote_qrels(
         """Abstract method to load relevance from remote dataset, to be overrode in child class.
 
         Args:
-            dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
-            split (str, optional): Split to load from the remote dataset. Defaults to :data:`'test'`.
-            save_dir (Optional[str], optional): Path to save the new downloaded relevance. Defaults to :data:`None`.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            split (str, optional): Split to load from the remote dataset. Defaults to ``'test'``.
+            save_dir (Optional[str], optional): Path to save the new downloaded relevance. Defaults to ``None``.
 
         Raises:
             NotImplementedError: Loading remote qrels is not implemented.
@@ -217,9 +217,9 @@ def _load_remote_queries(
         """Abstract method to load queries from remote dataset, to be overrode in child class.
 
         Args:
-            dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
-            split (str, optional): Split to load from the remote dataset. Defaults to :data:`'test'`.
-            save_dir (Optional[str], optional): Path to save the new downloaded queries. Defaults to :data:`None`.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            split (str, optional): Split to load from the remote dataset. Defaults to ``'test'``.
+            save_dir (Optional[str], optional): Path to save the new downloaded queries. Defaults to ``None``.
 
         Raises:
             NotImplementedError
@@ -234,7 +234,7 @@ def _load_local_corpus(self, save_dir: str, dataset_name: Optional[str] = None)
 
         Args:
             save_dir (str): Path to save the loaded corpus.
-            dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
 
         Returns:
             datasets.DatasetDict: A dict of corpus with id as key, title and text as value.
@@ -257,8 +257,8 @@ def _load_local_qrels(self, save_dir: str, dataset_name: Optional[str] = None, s
 
         Args:
             save_dir (str):  Path to save the loaded relevance.
-            dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`.
-            split (str, optional): Split to load from the local dataset. Defaults to :data:`'test'`.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            split (str, optional): Split to load from the local dataset. Defaults to ``'test'``.
 
         Raises:
             ValueError
@@ -292,8 +292,8 @@ def _load_local_queries(self, save_dir: str, dataset_name: Optional[str] = None,
 
         Args:
             save_dir (str):  Path to save the loaded queries.
-            dataset_name (Optional[str], optional): Name of the dataset. Defaults to None.
-            split (str, optional): Split to load from the local dataset. Defaults to 'test'.
+            dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``.
+            split (str, optional): Split to load from the local dataset. Defaults to ``'test'``.
 
         Raises:
             ValueError

FlagEmbedding/abc/evaluation/evaluator.py

@@ -357,7 +357,7 @@ def evaluate_results(
             k_values (List[int], optional): Cutoffs. Defaults to :data:`[1, 3, 5, 10, 100, 1000]`.
 
         Returns:
-            _type_: _description_
+            dict: Evaluation results.
         """
         eval_results_dict = {}
 

docs/source/API/abc/evaluation.rst

@@ -0,0 +1,9 @@
+Evaluation
+==========
+
+.. toctree::
+    evaluation/arguments
+    evaluation/data_loader
+    evaluation/searcher
+    evaluation/evaluator
+    evaluation/runner

docs/source/API/abc/evaluation/arguments.rst

@@ -0,0 +1,7 @@
+Arguments
+=========
+
+.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalArgs
+
+
+.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalModelArgs

docs/source/API/abc/evaluation/data_loader.rst

@@ -0,0 +1,25 @@
+dataset loader
+==============
+
+.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.available_dataset_names
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.available_splits
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.check_dataset_names
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.check_splits
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.load_corpus
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.load_qrels
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.load_queries
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_remote_corpus
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_remote_qrels
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_remote_queries
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_local_corpus
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_local_qrels
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_local_queries
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._download_file
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._get_fpath_size
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._download_gz_file
+.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._download_zip_file

docs/source/API/abc/evaluation/evaluator.rst

@@ -0,0 +1,4 @@
+Evaluator
+=========
+
+.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvaluator

docs/source/API/abc/evaluation/runner.rst

@@ -0,0 +1,4 @@
+runner
+======
+
+.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalRunner

docs/source/API/abc/evaluation/searcher.rst

@@ -0,0 +1,18 @@
+========
+searcher
+========
+
+EvalRetriever
+=============
+
+.. autoclass:: FlagEmbedding.abc.evaluation.EvalRetriever
+
+EvalDenseRetriever
+==================
+
+.. autoclass:: FlagEmbedding.abc.evaluation.EvalDenseRetriever
+
+EvalReranker
+============
+
+.. autoclass:: FlagEmbedding.abc.evaluation.EvalReranker