From 7caa5989a75fc9abd20a5acc99212683b0214fee Mon Sep 17 00:00:00 2001 From: ZiyiXia Date: Wed, 13 Nov 2024 20:03:21 +0800 Subject: [PATCH] eval abc --- FlagEmbedding/abc/evaluation/data_loader.py | 46 +++++++++---------- FlagEmbedding/abc/evaluation/evaluator.py | 2 +- docs/source/API/abc/evaluation.rst | 9 ++++ docs/source/API/abc/evaluation/arguments.rst | 7 +++ .../source/API/abc/evaluation/data_loader.rst | 25 ++++++++++ docs/source/API/abc/evaluation/evaluator.rst | 4 ++ docs/source/API/abc/evaluation/runner.rst | 4 ++ docs/source/API/abc/evaluation/searcher.rst | 18 ++++++++ 8 files changed, 91 insertions(+), 24 deletions(-) create mode 100644 docs/source/API/abc/evaluation.rst create mode 100644 docs/source/API/abc/evaluation/arguments.rst create mode 100644 docs/source/API/abc/evaluation/data_loader.rst create mode 100644 docs/source/API/abc/evaluation/evaluator.rst create mode 100644 docs/source/API/abc/evaluation/runner.rst create mode 100644 docs/source/API/abc/evaluation/searcher.rst diff --git a/FlagEmbedding/abc/evaluation/data_loader.py b/FlagEmbedding/abc/evaluation/data_loader.py index 4a73f2af..8dfddb98 100644 --- a/FlagEmbedding/abc/evaluation/data_loader.py +++ b/FlagEmbedding/abc/evaluation/data_loader.py @@ -17,10 +17,10 @@ class AbsEvalDataLoader(ABC): Args: eval_name (str): The experiment name of current evaluation. - dataset_dir (str, optional): path to the datasets. Defaults to :data:`None`. - cache_dir (str, optional): Path to HuggingFace cache directory. Defaults to :data:`None`. - token (str, optional): HF_TOKEN to access the private datasets/models in HF. Defaults to :data:`None`. - force_redownload: If True, will force redownload the dataset to cover the local dataset. Defaults to :data:`False`. + dataset_dir (str, optional): path to the datasets. Defaults to ``None``. + cache_dir (str, optional): Path to HuggingFace cache directory. Defaults to ``None``. + token (str, optional): HF_TOKEN to access the private datasets/models in HF. Defaults to ``None``. + force_redownload: If True, will force redownload the dataset to cover the local dataset. Defaults to ``False``. """ def __init__( self, @@ -78,7 +78,7 @@ def check_splits(self, splits: Union[str, List[str]], dataset_name: Optional[str Args: splits (Union[str, List[str]]): Splits to check. - dataset_name (Optional[str], optional): Name of dataset to check. Defaults to None. + dataset_name (Optional[str], optional): Name of dataset to check. Defaults to ``None``. Returns: List[str]: The available splits. @@ -98,7 +98,7 @@ def load_corpus(self, dataset_name: Optional[str] = None) -> datasets.DatasetDic """Load the corpus from the dataset. Args: - dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`. + dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``. Returns: datasets.DatasetDict: A dict of corpus with id as key, title and text as value. @@ -116,8 +116,8 @@ def load_qrels(self, dataset_name: Optional[str] = None, split: str = 'test') -> """Load the qrels from the dataset. Args: - dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`. - split (str, optional): The split to load relevance from. Defaults to :data:'test'. + dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``. + split (str, optional): The split to load relevance from. Defaults to ``'test'``. Raises: ValueError @@ -144,8 +144,8 @@ def load_queries(self, dataset_name: Optional[str] = None, split: str = 'test') """Load the queries from the dataset. Args: - dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`. - split (str, optional): The split to load queries from. Defaults to :data:`'test'`. + dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``. + split (str, optional): The split to load queries from. Defaults to ``'test'``. Raises: ValueError @@ -176,8 +176,8 @@ def _load_remote_corpus( """Abstract method to load corpus from remote dataset, to be overrode in child class. Args: - dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`. - save_dir (Optional[str], optional): Path to save the new downloaded corpus. Defaults to :data:`None`. + dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``. + save_dir (Optional[str], optional): Path to save the new downloaded corpus. Defaults to ``None``. Raises: NotImplementedError: Loading remote corpus is not implemented. @@ -196,9 +196,9 @@ def _load_remote_qrels( """Abstract method to load relevance from remote dataset, to be overrode in child class. Args: - dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`. - split (str, optional): Split to load from the remote dataset. Defaults to :data:`'test'`. - save_dir (Optional[str], optional): Path to save the new downloaded relevance. Defaults to :data:`None`. + dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``. + split (str, optional): Split to load from the remote dataset. Defaults to ``'test'``. + save_dir (Optional[str], optional): Path to save the new downloaded relevance. Defaults to ``None``. Raises: NotImplementedError: Loading remote qrels is not implemented. @@ -217,9 +217,9 @@ def _load_remote_queries( """Abstract method to load queries from remote dataset, to be overrode in child class. Args: - dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`. - split (str, optional): Split to load from the remote dataset. Defaults to :data:`'test'`. - save_dir (Optional[str], optional): Path to save the new downloaded queries. Defaults to :data:`None`. + dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``. + split (str, optional): Split to load from the remote dataset. Defaults to ``'test'``. + save_dir (Optional[str], optional): Path to save the new downloaded queries. Defaults to ``None``. Raises: NotImplementedError @@ -234,7 +234,7 @@ def _load_local_corpus(self, save_dir: str, dataset_name: Optional[str] = None) Args: save_dir (str): Path to save the loaded corpus. - dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`. + dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``. Returns: datasets.DatasetDict: A dict of corpus with id as key, title and text as value. @@ -257,8 +257,8 @@ def _load_local_qrels(self, save_dir: str, dataset_name: Optional[str] = None, s Args: save_dir (str): Path to save the loaded relevance. - dataset_name (Optional[str], optional): Name of the dataset. Defaults to :data:`None`. - split (str, optional): Split to load from the local dataset. Defaults to :data:`'test'`. + dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``. + split (str, optional): Split to load from the local dataset. Defaults to ``'test'``. Raises: ValueError @@ -292,8 +292,8 @@ def _load_local_queries(self, save_dir: str, dataset_name: Optional[str] = None, Args: save_dir (str): Path to save the loaded queries. - dataset_name (Optional[str], optional): Name of the dataset. Defaults to None. - split (str, optional): Split to load from the local dataset. Defaults to 'test'. + dataset_name (Optional[str], optional): Name of the dataset. Defaults to ``None``. + split (str, optional): Split to load from the local dataset. Defaults to ``'test'``. Raises: ValueError diff --git a/FlagEmbedding/abc/evaluation/evaluator.py b/FlagEmbedding/abc/evaluation/evaluator.py index c353b3c3..45c96e53 100644 --- a/FlagEmbedding/abc/evaluation/evaluator.py +++ b/FlagEmbedding/abc/evaluation/evaluator.py @@ -357,7 +357,7 @@ def evaluate_results( k_values (List[int], optional): Cutoffs. Defaults to :data:`[1, 3, 5, 10, 100, 1000]`. Returns: - _type_: _description_ + dict: Evaluation results. """ eval_results_dict = {} diff --git a/docs/source/API/abc/evaluation.rst b/docs/source/API/abc/evaluation.rst new file mode 100644 index 00000000..40896911 --- /dev/null +++ b/docs/source/API/abc/evaluation.rst @@ -0,0 +1,9 @@ +Evaluation +========== + +.. toctree:: + evaluation/arguments + evaluation/data_loader + evaluation/searcher + evaluation/evaluator + evaluation/runner \ No newline at end of file diff --git a/docs/source/API/abc/evaluation/arguments.rst b/docs/source/API/abc/evaluation/arguments.rst new file mode 100644 index 00000000..287e601e --- /dev/null +++ b/docs/source/API/abc/evaluation/arguments.rst @@ -0,0 +1,7 @@ +Arguments +========= + +.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalArgs + + +.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalModelArgs \ No newline at end of file diff --git a/docs/source/API/abc/evaluation/data_loader.rst b/docs/source/API/abc/evaluation/data_loader.rst new file mode 100644 index 00000000..38c3d8b2 --- /dev/null +++ b/docs/source/API/abc/evaluation/data_loader.rst @@ -0,0 +1,25 @@ +dataset loader +============== + +.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader + +Methods +------- + +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.available_dataset_names +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.available_splits +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.check_dataset_names +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.check_splits +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.load_corpus +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.load_qrels +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader.load_queries +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_remote_corpus +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_remote_qrels +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_remote_queries +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_local_corpus +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_local_qrels +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._load_local_queries +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._download_file +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._get_fpath_size +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._download_gz_file +.. automethod:: FlagEmbedding.abc.evaluation.AbsEvalDataLoader._download_zip_file \ No newline at end of file diff --git a/docs/source/API/abc/evaluation/evaluator.rst b/docs/source/API/abc/evaluation/evaluator.rst new file mode 100644 index 00000000..af3317ce --- /dev/null +++ b/docs/source/API/abc/evaluation/evaluator.rst @@ -0,0 +1,4 @@ +Evaluator +========= + +.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvaluator \ No newline at end of file diff --git a/docs/source/API/abc/evaluation/runner.rst b/docs/source/API/abc/evaluation/runner.rst new file mode 100644 index 00000000..b38f0b59 --- /dev/null +++ b/docs/source/API/abc/evaluation/runner.rst @@ -0,0 +1,4 @@ +runner +====== + +.. autoclass:: FlagEmbedding.abc.evaluation.AbsEvalRunner \ No newline at end of file diff --git a/docs/source/API/abc/evaluation/searcher.rst b/docs/source/API/abc/evaluation/searcher.rst new file mode 100644 index 00000000..f713ca4d --- /dev/null +++ b/docs/source/API/abc/evaluation/searcher.rst @@ -0,0 +1,18 @@ +======== +searcher +======== + +EvalRetriever +============= + +.. autoclass:: FlagEmbedding.abc.evaluation.EvalRetriever + +EvalDenseRetriever +================== + +.. autoclass:: FlagEmbedding.abc.evaluation.EvalDenseRetriever + +EvalReranker +============ + +.. autoclass:: FlagEmbedding.abc.evaluation.EvalReranker \ No newline at end of file