Move tests old (pydvl.value) vs. new (pydvl.valuation) to old testing…

… module to make

new testing module independent of pydvl.value

tests/valuation/methods/test_classwise_shapley.py

@@ -5,30 +5,19 @@
 import numpy as np
 import pytest
 from numpy.typing import NDArray
-from sklearn import datasets
-from sklearn.linear_model import LogisticRegression
 from typing_extensions import Self
 
-from pydvl.utils.dataset import Dataset as OldDataset
-from pydvl.utils.utility import Utility as OldUtility
 from pydvl.valuation.dataset import Dataset
 from pydvl.valuation.methods import ClasswiseShapleyValuation
 from pydvl.valuation.result import ValuationResult
 from pydvl.valuation.samplers import (
     ClasswiseSampler,
     DeterministicPermutationSampler,
     DeterministicUniformSampler,
-    PermutationSampler,
-    UniformSampler,
 )
 from pydvl.valuation.scorers import ClasswiseSupervisedScorer
 from pydvl.valuation.stopping import MaxUpdates
 from pydvl.valuation.utility import ClasswiseModelUtility
-from pydvl.value import MaxChecks
-from pydvl.value.shapley.classwise import ClasswiseScorer as OldClasswiseScorer
-from pydvl.value.shapley.classwise import compute_classwise_shapley_values
-from pydvl.value.shapley.truncated import NoTruncation
-from tests.value import check_values
 
 from .. import check_values
 
@@ -173,49 +162,3 @@ def test_classwise_shapley(
     valuation.fit(train_dataset_manual_derivation)
     values = valuation.values()
     check_values(values, exact_solution, **check_kwargs)
-
-
-@pytest.mark.flaky(reruns=2)
-@pytest.mark.parametrize("n_samples", [500], ids=lambda x: "n_samples={}".format(x))
-def test_old_vs_new(
-    n_samples: int,
-    seed,
-):
-    model = LogisticRegression(random_state=seed)
-    old_data = OldDataset.from_sklearn(
-        datasets.load_iris(),
-        train_size=0.05,
-        random_state=seed,
-        stratify_by_target=True,
-    )
-    old_scorer = OldClasswiseScorer("accuracy", initial_label=0)
-    old_u = OldUtility(model=model, data=old_data, scorer=old_scorer)
-    old_values = compute_classwise_shapley_values(
-        old_u,
-        done=MaxChecks(n_samples),
-        truncation=NoTruncation(),
-        done_sample_complements=MaxChecks(1),
-        seed=seed,
-    )
-
-    new_train_data = Dataset(old_data.x_train, old_data.y_train)
-    new_test_data = Dataset(old_data.x_test, old_data.y_test)
-
-    in_class_sampler = PermutationSampler(seed=seed)
-    out_of_class_sampler = UniformSampler(seed=seed)
-    sampler = ClasswiseSampler(
-        in_class=in_class_sampler,
-        out_of_class=out_of_class_sampler,
-    )
-    new_u = ClasswiseModelUtility(
-        model,
-        ClasswiseSupervisedScorer("accuracy", new_test_data),
-        catch_errors=False,
-    )
-    valuation = ClasswiseShapleyValuation(
-        new_u,
-        sampler=sampler,
-        is_done=MaxUpdates(n_samples),
-    )
-    valuation.fit(new_train_data)
-    check_values(valuation.values(), old_values, atol=1e-1, rtol=1e-1)

tests/valuation/methods/test_knn_shapley.py

@@ -4,14 +4,11 @@
 from sklearn import datasets
 from sklearn.neighbors import KNeighborsClassifier
 
-from pydvl.utils.dataset import Dataset as OldDataset
-from pydvl.utils.utility import Utility as OldUtility
 from pydvl.valuation.dataset import Dataset
 from pydvl.valuation.methods import DataShapleyValuation, KNNShapleyValuation
 from pydvl.valuation.samplers import PermutationSampler
 from pydvl.valuation.stopping import MinUpdates
 from pydvl.valuation.utility import KNNClassifierUtility
-from pydvl.value.shapley.knn import knn_shapley as old_knn_shapley
 
 
 @pytest.fixture(scope="module")
@@ -52,23 +49,3 @@ def test_against_montecarlo(n_jobs, data, montecarlo_results):
     np.testing.assert_allclose(
         results.values, montecarlo_results.values, atol=1e-2, rtol=1e-2
     )
-
-
-@pytest.mark.xfail(reason="Suspected bug in old implementation.")
-def test_old_vs_new(seed, data):
-    model = KNeighborsClassifier(n_neighbors=5)
-    old_data = OldDataset.from_sklearn(
-        datasets.load_iris(),
-        train_size=0.05,
-        random_state=seed,
-        stratify_by_target=True,
-    )
-    old_u = OldUtility(model=model, data=old_data)
-    old_values = old_knn_shapley(old_u, progress=False).values
-
-    data_train, data_test = data
-    utility = KNNClassifierUtility(model=model, test_data=data_test)
-    new_valuation = KNNShapleyValuation(utility, progress=False)
-    new_values = new_valuation.fit(data_train).values().values
-
-    np.testing.assert_allclose(new_values, old_values, atol=1e-2, rtol=1e-2)

tests/value/shapley/test_classwise.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import Dict, Tuple, cast
 
 import numpy as np
@@ -6,9 +8,32 @@
 import sklearn
 from numpy.typing import NDArray
 from packaging import version
-
-from pydvl.utils import Dataset, Utility, powerset
-from pydvl.value import MaxChecks, ValuationResult
+from sklearn import datasets
+from sklearn.linear_model import LogisticRegression
+from value import check_values
+
+from pydvl.utils import Dataset
+from pydvl.utils import Dataset as OldDataset
+from pydvl.utils import Utility
+from pydvl.utils import Utility as OldUtility
+from pydvl.utils import powerset
+from pydvl.valuation import (
+    ClasswiseModelUtility,
+    ClasswiseSampler,
+    ClasswiseShapleyValuation,
+    ClasswiseSupervisedScorer,
+    Dataset,
+    PermutationSampler,
+    UniformSampler,
+)
+from pydvl.valuation.stopping import MaxUpdates
+from pydvl.value import ClasswiseScorer as OldClasswiseScorer
+from pydvl.value import (
+    MaxChecks,
+    NoTruncation,
+    ValuationResult,
+    compute_classwise_shapley_values,
+)
 from pydvl.value.shapley.classwise import (
     ClasswiseScorer,
     compute_classwise_shapley_values,
@@ -430,3 +455,49 @@ def dataset_left_right_margins(
     y = y.astype(int)
     x = np.expand_dims(x, -1)
     return x, y, {"left_margin": left_margin, "right_margin": right_margin}
+
+
+@pytest.mark.flaky(reruns=2)
+@pytest.mark.parametrize("n_samples", [500], ids=lambda x: "n_samples={}".format(x))
+def test_old_vs_new(
+    n_samples: int,
+    seed,
+):
+    model = LogisticRegression(random_state=seed)
+    old_data = OldDataset.from_sklearn(
+        datasets.load_iris(),
+        train_size=0.05,
+        random_state=seed,
+        stratify_by_target=True,
+    )
+    old_scorer = OldClasswiseScorer("accuracy", initial_label=0)
+    old_u = OldUtility(model=model, data=old_data, scorer=old_scorer)
+    old_values = compute_classwise_shapley_values(
+        old_u,
+        done=MaxChecks(n_samples),
+        truncation=NoTruncation(),
+        done_sample_complements=MaxChecks(1),
+        seed=seed,
+    )
+
+    new_train_data = Dataset(old_data.x_train, old_data.y_train)
+    new_test_data = Dataset(old_data.x_test, old_data.y_test)
+
+    in_class_sampler = PermutationSampler(seed=seed)
+    out_of_class_sampler = UniformSampler(seed=seed)
+    sampler = ClasswiseSampler(
+        in_class=in_class_sampler,
+        out_of_class=out_of_class_sampler,
+    )
+    new_u = ClasswiseModelUtility(
+        model,
+        ClasswiseSupervisedScorer("accuracy", new_test_data),
+        catch_errors=False,
+    )
+    valuation = ClasswiseShapleyValuation(
+        new_u,
+        sampler=sampler,
+        is_done=MaxUpdates(n_samples),
+    )
+    valuation.fit(new_train_data)
+    check_values(valuation.values(), old_values, atol=1e-1, rtol=1e-1)

tests/value/shapley/test_knn.py

@@ -1,14 +1,19 @@
 import logging
 
 import numpy as np
+import pytest
 from sklearn import datasets
 from sklearn.metrics import make_scorer
 from sklearn.neighbors import KNeighborsClassifier
 
 from pydvl.parallel.backend import available_cpus
+from pydvl.utils import Dataset as OldDataset
+from pydvl.utils import Utility as OldUtility
 from pydvl.utils.dataset import Dataset
 from pydvl.utils.score import Scorer
 from pydvl.utils.utility import Utility
+from pydvl.valuation import KNNClassifierUtility, KNNShapleyValuation
+from pydvl.value import knn_shapley as old_knn_shapley
 from pydvl.value.shapley.knn import knn_shapley
 from pydvl.value.shapley.naive import combinatorial_exact_shapley
 
@@ -56,3 +61,23 @@ def knn_loss_function(labels, predictions, n_classes=3):
     top_knn = knn_values.indices[-2:]
     top_exact = exact_values.indices[-4:]
     assert np.all([k in top_exact for k in top_knn])
+
+
+@pytest.mark.xfail(reason="Suspected bug in old implementation.")
+def test_old_vs_new(seed, data):
+    model = KNeighborsClassifier(n_neighbors=5)
+    old_data = OldDataset.from_sklearn(
+        datasets.load_iris(),
+        train_size=0.05,
+        random_state=seed,
+        stratify_by_target=True,
+    )
+    old_u = OldUtility(model=model, data=old_data)
+    old_values = old_knn_shapley(old_u, progress=False).values
+
+    data_train, data_test = data
+    utility = KNNClassifierUtility(model=model, test_data=data_test)
+    new_valuation = KNNShapleyValuation(utility, progress=False)
+    new_values = new_valuation.fit(data_train).values().values
+
+    np.testing.assert_allclose(new_values, old_values, atol=1e-2, rtol=1e-2)