Adds PipeEvaluator

README.md

@@ -115,6 +115,12 @@ The non interactive evaluators also supports `--workers N` to run in the evaluat
 $ bench run --evaluator string-match --workers 5
 ```
 
+It is also possible to combine multiple evaluators, e.g. first do string-match, and only if that doe not match do semantic similarity. This speeds up the evaluation process and can help save on API calls.
+
+```bash
+$ bench run --evaluator string-match,embedding,semantic
+```
+
 To accelerate the evaluation process, BenchLLM uses a cache. If a (prediction, expected) pair has been evaluated in the past and a cache was used, the evaluation output will be saved for future evaluations. There are several types of caches:
 
 - `memory`, only caches output values during the current run. This is particularly useful when running with `--retry-count N`

benchllm/cli/utils.py

@@ -6,6 +6,7 @@
 from benchllm.evaluator import (
     EmbeddingEvaluator,
     Evaluator,
+    PipeEvaluator,
     SemanticEvaluator,
     StringMatchEvaluator,
 )
@@ -24,6 +25,11 @@ def output_dir_factory() -> Path:
 
 
 def get_evaluator(evaluator_name: str, model: str, workers: int) -> Evaluator:
+    if "," in evaluator_name:
+        evaluator_names = evaluator_name.split(",")
+        return PipeEvaluator(
+            workers=workers, evaluators=[get_evaluator(name, model, workers) for name in evaluator_names]
+        )
     if evaluator_name == "semantic":
         return SemanticEvaluator(model=model, workers=workers)
     elif evaluator_name == "interactive":

benchllm/evaluator/__init__.py

@@ -1,5 +1,12 @@
-from benchllm.evaluator.evaluator import Evaluator  # noqa
 # Adding an empty comment to force import order to avoid circular imports
+from benchllm.evaluator.evaluator import Evaluator  # noqa
+
+
+def noop():
+    pass
+
+
 from benchllm.evaluator.embedding import EmbeddingEvaluator  # noqa
+from benchllm.evaluator.pipe import PipeEvaluator
 from benchllm.evaluator.semantic import SemanticEvaluator  # noqa
 from benchllm.evaluator.string_match import StringMatchEvaluator  # noqa

benchllm/evaluator/pipe.py

@@ -0,0 +1,19 @@
+from benchllm.data_types import Prediction
+from benchllm.evaluator import Evaluator
+
+
+class PipeEvaluator(Evaluator):
+    def __init__(self, evaluators: list[Evaluator], workers: int = 1):
+        super().__init__(workers=workers)
+        self._evaluators: list[Evaluator] = evaluators
+
+    def evaluate_prediction(self, prediction: Prediction) -> list[Evaluator.Candidate]:
+        candidates = []
+        for evaluator in self._evaluators:
+            candidates = evaluator.evaluate_prediction(prediction)
+            # only return passed candidates so negative predication don't get cached
+            # since they might have passed further down the chain
+            passed_candidates = [candidate for candidate in candidates if candidate.passed]
+            if passed_candidates:
+                return passed_candidates
+        return candidates

test/evaulator/test_pipe.py

@@ -0,0 +1,53 @@
+from benchllm import Prediction, Test
+from benchllm.data_types import FunctionID
+from benchllm.evaluator import Evaluator, PipeEvaluator
+
+PREDICTION_EXAMPLE = Prediction(
+    test=Test(input="Who are you?", expected=["Yoda I am.", "Yoda"]),
+    output="I am Yoda.",
+    time_elapsed=0,
+    function_id=FunctionID.default(),
+)
+
+
+class HardCoded(Evaluator):
+    def __init__(self, passes: list[bool]):
+        super().__init__(workers=1)
+        self.passes = passes
+
+    def evaluate_prediction(self, prediction: Prediction) -> list[Evaluator.Candidate]:
+        assert len(self.passes) == len(prediction.test.expected)
+        candidates = []
+        for passed, expected in zip(self.passes, prediction.test.expected):
+            candidates.append(
+                Evaluator.Candidate(
+                    prediction=prediction.output, expected=expected, score=1.0 if passed else 0, passed=passed
+                )
+            )
+        return candidates
+
+
+def test_evaluator_runs_all_evaluators():
+    pipe = PipeEvaluator([HardCoded([False, False]), HardCoded([True, False])])
+    pipe.load([PREDICTION_EXAMPLE])
+    evaluations = pipe.run()
+    assert len(evaluations) == 1
+    assert evaluations[0].passed
+
+
+def test_evaluator_runs_all_evaluators_with_all_failures():
+    pipe = PipeEvaluator([HardCoded([False, False]), HardCoded([False, False])])
+    pipe.load([PREDICTION_EXAMPLE])
+
+    evaluations = pipe.run()
+    assert len(evaluations) == 1
+    assert not evaluations[0].passed
+
+
+def test_evaluator_runs_all_evaluators_with_early_stopping():
+    pipe = PipeEvaluator([HardCoded([False, True]), HardCoded([])])
+    pipe.load([PREDICTION_EXAMPLE])
+
+    evaluations = pipe.run()
+    assert len(evaluations) == 1
+    assert evaluations[0].passed