diff --git a/tests/valuation/methods/test_classwise_shapley.py b/tests/valuation/methods/test_classwise_shapley.py index 61ccbafb8..d55288750 100644 --- a/tests/valuation/methods/test_classwise_shapley.py +++ b/tests/valuation/methods/test_classwise_shapley.py @@ -5,12 +5,8 @@ import numpy as np import pytest from numpy.typing import NDArray -from sklearn import datasets -from sklearn.linear_model import LogisticRegression from typing_extensions import Self -from pydvl.utils.dataset import Dataset as OldDataset -from pydvl.utils.utility import Utility as OldUtility from pydvl.valuation.dataset import Dataset from pydvl.valuation.methods import ClasswiseShapleyValuation from pydvl.valuation.result import ValuationResult @@ -18,17 +14,10 @@ ClasswiseSampler, DeterministicPermutationSampler, DeterministicUniformSampler, - PermutationSampler, - UniformSampler, ) from pydvl.valuation.scorers import ClasswiseSupervisedScorer from pydvl.valuation.stopping import MaxUpdates from pydvl.valuation.utility import ClasswiseModelUtility -from pydvl.value import MaxChecks -from pydvl.value.shapley.classwise import ClasswiseScorer as OldClasswiseScorer -from pydvl.value.shapley.classwise import compute_classwise_shapley_values -from pydvl.value.shapley.truncated import NoTruncation -from tests.value import check_values from .. import check_values @@ -173,49 +162,3 @@ def test_classwise_shapley( valuation.fit(train_dataset_manual_derivation) values = valuation.values() check_values(values, exact_solution, **check_kwargs) - - -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize("n_samples", [500], ids=lambda x: "n_samples={}".format(x)) -def test_old_vs_new( - n_samples: int, - seed, -): - model = LogisticRegression(random_state=seed) - old_data = OldDataset.from_sklearn( - datasets.load_iris(), - train_size=0.05, - random_state=seed, - stratify_by_target=True, - ) - old_scorer = OldClasswiseScorer("accuracy", initial_label=0) - old_u = OldUtility(model=model, data=old_data, scorer=old_scorer) - old_values = compute_classwise_shapley_values( - old_u, - done=MaxChecks(n_samples), - truncation=NoTruncation(), - done_sample_complements=MaxChecks(1), - seed=seed, - ) - - new_train_data = Dataset(old_data.x_train, old_data.y_train) - new_test_data = Dataset(old_data.x_test, old_data.y_test) - - in_class_sampler = PermutationSampler(seed=seed) - out_of_class_sampler = UniformSampler(seed=seed) - sampler = ClasswiseSampler( - in_class=in_class_sampler, - out_of_class=out_of_class_sampler, - ) - new_u = ClasswiseModelUtility( - model, - ClasswiseSupervisedScorer("accuracy", new_test_data), - catch_errors=False, - ) - valuation = ClasswiseShapleyValuation( - new_u, - sampler=sampler, - is_done=MaxUpdates(n_samples), - ) - valuation.fit(new_train_data) - check_values(valuation.values(), old_values, atol=1e-1, rtol=1e-1) diff --git a/tests/valuation/methods/test_knn_shapley.py b/tests/valuation/methods/test_knn_shapley.py index c5c8ad4df..e8af2e8f2 100644 --- a/tests/valuation/methods/test_knn_shapley.py +++ b/tests/valuation/methods/test_knn_shapley.py @@ -4,14 +4,11 @@ from sklearn import datasets from sklearn.neighbors import KNeighborsClassifier -from pydvl.utils.dataset import Dataset as OldDataset -from pydvl.utils.utility import Utility as OldUtility from pydvl.valuation.dataset import Dataset from pydvl.valuation.methods import DataShapleyValuation, KNNShapleyValuation from pydvl.valuation.samplers import PermutationSampler from pydvl.valuation.stopping import MinUpdates from pydvl.valuation.utility import KNNClassifierUtility -from pydvl.value.shapley.knn import knn_shapley as old_knn_shapley @pytest.fixture(scope="module") @@ -52,23 +49,3 @@ def test_against_montecarlo(n_jobs, data, montecarlo_results): np.testing.assert_allclose( results.values, montecarlo_results.values, atol=1e-2, rtol=1e-2 ) - - -@pytest.mark.xfail(reason="Suspected bug in old implementation.") -def test_old_vs_new(seed, data): - model = KNeighborsClassifier(n_neighbors=5) - old_data = OldDataset.from_sklearn( - datasets.load_iris(), - train_size=0.05, - random_state=seed, - stratify_by_target=True, - ) - old_u = OldUtility(model=model, data=old_data) - old_values = old_knn_shapley(old_u, progress=False).values - - data_train, data_test = data - utility = KNNClassifierUtility(model=model, test_data=data_test) - new_valuation = KNNShapleyValuation(utility, progress=False) - new_values = new_valuation.fit(data_train).values().values - - np.testing.assert_allclose(new_values, old_values, atol=1e-2, rtol=1e-2) diff --git a/tests/value/shapley/test_classwise.py b/tests/value/shapley/test_classwise.py index 6876394b2..ab7ae8682 100644 --- a/tests/value/shapley/test_classwise.py +++ b/tests/value/shapley/test_classwise.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Dict, Tuple, cast import numpy as np @@ -6,9 +8,32 @@ import sklearn from numpy.typing import NDArray from packaging import version - -from pydvl.utils import Dataset, Utility, powerset -from pydvl.value import MaxChecks, ValuationResult +from sklearn import datasets +from sklearn.linear_model import LogisticRegression +from value import check_values + +from pydvl.utils import Dataset +from pydvl.utils import Dataset as OldDataset +from pydvl.utils import Utility +from pydvl.utils import Utility as OldUtility +from pydvl.utils import powerset +from pydvl.valuation import ( + ClasswiseModelUtility, + ClasswiseSampler, + ClasswiseShapleyValuation, + ClasswiseSupervisedScorer, + Dataset, + PermutationSampler, + UniformSampler, +) +from pydvl.valuation.stopping import MaxUpdates +from pydvl.value import ClasswiseScorer as OldClasswiseScorer +from pydvl.value import ( + MaxChecks, + NoTruncation, + ValuationResult, + compute_classwise_shapley_values, +) from pydvl.value.shapley.classwise import ( ClasswiseScorer, compute_classwise_shapley_values, @@ -430,3 +455,49 @@ def dataset_left_right_margins( y = y.astype(int) x = np.expand_dims(x, -1) return x, y, {"left_margin": left_margin, "right_margin": right_margin} + + +@pytest.mark.flaky(reruns=2) +@pytest.mark.parametrize("n_samples", [500], ids=lambda x: "n_samples={}".format(x)) +def test_old_vs_new( + n_samples: int, + seed, +): + model = LogisticRegression(random_state=seed) + old_data = OldDataset.from_sklearn( + datasets.load_iris(), + train_size=0.05, + random_state=seed, + stratify_by_target=True, + ) + old_scorer = OldClasswiseScorer("accuracy", initial_label=0) + old_u = OldUtility(model=model, data=old_data, scorer=old_scorer) + old_values = compute_classwise_shapley_values( + old_u, + done=MaxChecks(n_samples), + truncation=NoTruncation(), + done_sample_complements=MaxChecks(1), + seed=seed, + ) + + new_train_data = Dataset(old_data.x_train, old_data.y_train) + new_test_data = Dataset(old_data.x_test, old_data.y_test) + + in_class_sampler = PermutationSampler(seed=seed) + out_of_class_sampler = UniformSampler(seed=seed) + sampler = ClasswiseSampler( + in_class=in_class_sampler, + out_of_class=out_of_class_sampler, + ) + new_u = ClasswiseModelUtility( + model, + ClasswiseSupervisedScorer("accuracy", new_test_data), + catch_errors=False, + ) + valuation = ClasswiseShapleyValuation( + new_u, + sampler=sampler, + is_done=MaxUpdates(n_samples), + ) + valuation.fit(new_train_data) + check_values(valuation.values(), old_values, atol=1e-1, rtol=1e-1) diff --git a/tests/value/shapley/test_knn.py b/tests/value/shapley/test_knn.py index cf935f347..dd0d5f79b 100644 --- a/tests/value/shapley/test_knn.py +++ b/tests/value/shapley/test_knn.py @@ -1,14 +1,19 @@ import logging import numpy as np +import pytest from sklearn import datasets from sklearn.metrics import make_scorer from sklearn.neighbors import KNeighborsClassifier from pydvl.parallel.backend import available_cpus +from pydvl.utils import Dataset as OldDataset +from pydvl.utils import Utility as OldUtility from pydvl.utils.dataset import Dataset from pydvl.utils.score import Scorer from pydvl.utils.utility import Utility +from pydvl.valuation import KNNClassifierUtility, KNNShapleyValuation +from pydvl.value import knn_shapley as old_knn_shapley from pydvl.value.shapley.knn import knn_shapley from pydvl.value.shapley.naive import combinatorial_exact_shapley @@ -56,3 +61,23 @@ def knn_loss_function(labels, predictions, n_classes=3): top_knn = knn_values.indices[-2:] top_exact = exact_values.indices[-4:] assert np.all([k in top_exact for k in top_knn]) + + +@pytest.mark.xfail(reason="Suspected bug in old implementation.") +def test_old_vs_new(seed, data): + model = KNeighborsClassifier(n_neighbors=5) + old_data = OldDataset.from_sklearn( + datasets.load_iris(), + train_size=0.05, + random_state=seed, + stratify_by_target=True, + ) + old_u = OldUtility(model=model, data=old_data) + old_values = old_knn_shapley(old_u, progress=False).values + + data_train, data_test = data + utility = KNNClassifierUtility(model=model, test_data=data_test) + new_valuation = KNNShapleyValuation(utility, progress=False) + new_values = new_valuation.fit(data_train).values().values + + np.testing.assert_allclose(new_values, old_values, atol=1e-2, rtol=1e-2)