diff --git a/.test_durations b/.test_durations index 7a7768311..78f10ff83 100644 --- a/.test_durations +++ b/.test_durations @@ -1,74 +1,74 @@ { - "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv1d_nn_pert]": 2.59026943400022, - "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv1d_nn_up]": 2.7703545530002884, - "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv1d_no_grad_up]": 0.8260756999989098, - "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv2d_nn_pert]": 1.101015895999808, - "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv2d_nn_up]": 1.206421760000012, - "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv3d_nn_pert]": 1.4294998579989624, - "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv3d_nn_up]": 1.3345100419992377, - "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[simple_nn_class_up]": 3.361096810000163, - "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[simple_nn_pert]": 0.6431655560008949, - "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[simple_nn_up]": 0.7108467549987836, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_pert-arnoldi]": 1.4143697240015172, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_pert-cg]": 2.522983850998571, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_pert-direct]": 1.3974800130017684, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_up-arnoldi]": 1.4222584220005956, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_up-cg]": 2.5742563249987143, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_up-direct]": 1.3653277730008995, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_no_grad_up-arnoldi]": 0.48600830500072334, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_no_grad_up-cg]": 0.7124692380002671, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_no_grad_up-direct]": 0.47575023000172223, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_pert-arnoldi]": 0.8454596849987865, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_pert-cg]": 1.7426123529985489, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_pert-direct]": 0.808057442000063, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_up-arnoldi]": 0.8408936979994905, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_up-cg]": 1.8711466349977854, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_up-direct]": 0.7968461060008849, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_pert-arnoldi]": 1.041476223997961, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_pert-cg]": 2.6348945509980695, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_pert-direct]": 1.0208977649999724, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_up-arnoldi]": 1.3290127370019036, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_up-cg]": 5.805227180999282, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_up-direct]": 1.8304335940010787, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_class_up-arnoldi]": 1.9109577300005185, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_class_up-cg]": 4.174298836998787, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_class_up-direct]": 1.5329143839990138, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_pert-arnoldi]": 0.4525704900006531, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_pert-cg]": 0.8970914879992051, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_pert-direct]": 0.46585072099878744, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_up-arnoldi]": 0.4456351110020478, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_up-cg]": 1.0693235140006436, - "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_up-direct]": 0.473094435999883, - "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv1d_nn_pert]": 2.9761773999980505, - "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv1d_nn_up]": 4.120701600999382, - "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv1d_no_grad_up]": 1.3337201610011107, - "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv2d_nn_pert]": 2.1662617799993313, - "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv2d_nn_up]": 3.132741712999632, - "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv3d_nn_pert]": 2.958187670999905, - "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv3d_nn_up]": 29.53393912699903, - "tests/influence/test_influence_calculator.py::test_dask_influence_nn[simple_nn_class_up]": 3.257567571998152, - "tests/influence/test_influence_calculator.py::test_dask_influence_nn[simple_nn_pert]": 1.361139677999745, - "tests/influence/test_influence_calculator.py::test_dask_influence_nn[simple_nn_up]": 1.261350679998941, - "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv1d_nn_pert]": 2.6579838110010314, - "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv1d_nn_up]": 2.6499502710012166, - "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv1d_no_grad_up]": 0.8881425300005503, - "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv2d_nn_pert]": 1.463408392999554, - "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv2d_nn_up]": 1.4602782740003022, - "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv3d_nn_pert]": 1.7320480180023878, - "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv3d_nn_up]": 1.5744405670029664, - "tests/influence/test_influence_calculator.py::test_sequential_calculator[simple_nn_class_up]": 4.504372877998321, - "tests/influence/test_influence_calculator.py::test_sequential_calculator[simple_nn_pert]": 0.8736393959989073, - "tests/influence/test_influence_calculator.py::test_sequential_calculator[simple_nn_up]": 0.8922971840001992, - "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv1d_nn_pert]": 2.381483594999736, - "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv1d_nn_up]": 2.314768557000207, - "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv1d_no_grad_up]": 0.7438636890019552, - "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv2d_nn_pert]": 0.9980942529964523, - "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv2d_nn_up]": 1.1705565329993988, - "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv3d_nn_pert]": 1.2230443010012095, - "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv3d_nn_up]": 4.6594328910014156, - "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[simple_nn_class_up]": 3.0931850600009057, - "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[simple_nn_pert]": 0.7189972920023138, - "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[simple_nn_up]": 0.7615732119993481, + "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv1d_nn_pert]": 1.1757412470178679, + "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv1d_nn_up]": 1.1531428449961822, + "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv1d_no_grad_up]": 0.40105982599197887, + "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv2d_nn_pert]": 0.49755765599547885, + "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv2d_nn_up]": 0.5384459960041568, + "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv3d_nn_pert]": 0.5693707920436282, + "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[conv3d_nn_up]": 0.5791179630032275, + "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[simple_nn_class_up]": 1.515325879008742, + "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[simple_nn_pert]": 0.31648814197978936, + "tests/influence/test_influence_calculator.py::test_dask_ekfac_influence[simple_nn_up]": 0.345498406997649, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_pert-arnoldi]": 0.7486136490188073, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_pert-cg]": 2.3709864970005583, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_pert-direct]": 0.6317156740115024, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_up-arnoldi]": 0.6990832140145358, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_up-cg]": 2.5801909349975176, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_nn_up-direct]": 0.6321751869982108, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_no_grad_up-arnoldi]": 0.3029264329816215, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_no_grad_up-cg]": 1.2931301009957679, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv1d_no_grad_up-direct]": 0.2809486609767191, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_pert-arnoldi]": 0.5121201520087197, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_pert-cg]": 1.8714381649915595, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_pert-direct]": 0.3564423870120663, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_up-arnoldi]": 0.4797562639869284, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_up-cg]": 1.9824987890315242, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv2d_nn_up-direct]": 0.34359909396152943, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_pert-arnoldi]": 0.5911412620043848, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_pert-cg]": 1.8817940490262117, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_pert-direct]": 0.40603021200513467, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_up-arnoldi]": 0.5107649029814638, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_up-cg]": 3.2007382639567368, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[conv3d_nn_up-direct]": 0.43204710801364854, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_class_up-arnoldi]": 1.2488551850256044, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_class_up-cg]": 16.231199133966584, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_class_up-direct]": 0.8788644630112685, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_pert-arnoldi]": 0.3679435200174339, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_pert-cg]": 1.6276155139494222, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_pert-direct]": 0.3059443189704325, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_up-arnoldi]": 0.29977179697016254, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_up-cg]": 1.5808336699556094, + "tests/influence/test_influence_calculator.py::test_dask_influence_factors[simple_nn_up-direct]": 0.2595673779724166, + "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv1d_nn_pert]": 1.648073843010934, + "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv1d_nn_up]": 2.2320262680004817, + "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv1d_no_grad_up]": 0.739165420003701, + "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv2d_nn_pert]": 1.3300797659903765, + "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv2d_nn_up]": 1.7796139150159433, + "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv3d_nn_pert]": 1.5524548839603085, + "tests/influence/test_influence_calculator.py::test_dask_influence_nn[conv3d_nn_up]": 15.148636500030989, + "tests/influence/test_influence_calculator.py::test_dask_influence_nn[simple_nn_class_up]": 2.0765505130111706, + "tests/influence/test_influence_calculator.py::test_dask_influence_nn[simple_nn_pert]": 0.7969291630142834, + "tests/influence/test_influence_calculator.py::test_dask_influence_nn[simple_nn_up]": 0.7292251750186551, + "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv1d_nn_pert]": 1.2291264449886512, + "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv1d_nn_up]": 1.24865967099322, + "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv1d_no_grad_up]": 0.5143154310062528, + "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv2d_nn_pert]": 0.7841348430083599, + "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv2d_nn_up]": 0.8018321000272408, + "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv3d_nn_pert]": 0.8897617959883064, + "tests/influence/test_influence_calculator.py::test_sequential_calculator[conv3d_nn_up]": 0.7785523339698557, + "tests/influence/test_influence_calculator.py::test_sequential_calculator[simple_nn_class_up]": 2.6527095659985207, + "tests/influence/test_influence_calculator.py::test_sequential_calculator[simple_nn_pert]": 0.5539844660379458, + "tests/influence/test_influence_calculator.py::test_sequential_calculator[simple_nn_up]": 0.5878762130159885, + "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv1d_nn_pert]": 0.9878947690012865, + "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv1d_nn_up]": 0.9868685769906733, + "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv1d_no_grad_up]": 0.31182356498902664, + "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv2d_nn_pert]": 0.38766848100931384, + "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv2d_nn_up]": 0.4189586569846142, + "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv3d_nn_pert]": 0.5046580989728682, + "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[conv3d_nn_up]": 2.282171113009099, + "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[simple_nn_class_up]": 1.2807428169762716, + "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[simple_nn_pert]": 0.2861086849879939, + "tests/influence/test_influence_calculator.py::test_thread_safety_violation_error[simple_nn_up]": 0.3151668659411371, "tests/influence/test_influences.py::test_influence_linear_model[cg-train_set_size_200-perturbation]": 0.8664472580130678, "tests/influence/test_influences.py::test_influence_linear_model[cg-train_set_size_200-up]": 0.18988716599415056, "tests/influence/test_influences.py::test_influence_linear_model[direct-train_set_size_200-perturbation]": 0.66577532098745, @@ -148,59 +148,62 @@ "tests/influence/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data3-8-160-1e-05]": 4.422049004002474, "tests/influence/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data4-4-250-1e-05]": 9.08382142597111, "tests/influence/test_util.py::test_lanzcos_low_rank_hessian_approx_exception": 0.0035210640053264797, - "tests/influence/torch/test_functional.py::test_get_hessian[model_data0-4-avg]": 0.20045989400023245, - "tests/influence/torch/test_functional.py::test_get_hessian[model_data0-4-full]": 0.06902083099885203, - "tests/influence/torch/test_functional.py::test_get_hessian[model_data1-5-avg]": 0.5016348780009139, - "tests/influence/torch/test_functional.py::test_get_hessian[model_data1-5-full]": 0.1801713530021516, - "tests/influence/torch/test_functional.py::test_get_hessian[model_data2-10-avg]": 0.1689359069987404, - "tests/influence/torch/test_functional.py::test_get_hessian[model_data2-10-full]": 0.06361526499858883, - "tests/influence/torch/test_functional.py::test_get_hessian[model_data3-8-avg]": 0.23464886900001147, - "tests/influence/torch/test_functional.py::test_get_hessian[model_data3-8-full]": 0.08587454999906186, - "tests/influence/torch/test_functional.py::test_get_hessian[model_data4-4-avg]": 3.4049244679990807, - "tests/influence/torch/test_functional.py::test_get_hessian[model_data4-4-full]": 0.8161465619996306, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data0-4-avg-no_precomputed_grad]": 0.07086462199913512, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data0-4-avg-precomputed_grad]": 0.05853749299967603, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data0-4-full-no_precomputed_grad]": 0.042555562999041285, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data0-4-full-precomputed_grad]": 0.04254312700140872, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data1-5-avg-no_precomputed_grad]": 0.12398883400055638, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data1-5-avg-precomputed_grad]": 0.09578048600087641, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data1-5-full-no_precomputed_grad]": 0.08458008900015557, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data1-5-full-precomputed_grad]": 0.08726069599833863, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data2-10-avg-no_precomputed_grad]": 0.04802275299880421, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data2-10-avg-precomputed_grad]": 0.03521194200220634, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data2-10-full-no_precomputed_grad]": 0.036551941999277915, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data2-10-full-precomputed_grad]": 0.037776481000037165, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data3-8-avg-no_precomputed_grad]": 0.07563198299976648, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data3-8-avg-precomputed_grad]": 0.06490736200066749, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data3-8-full-no_precomputed_grad]": 0.04830970400143997, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data3-8-full-precomputed_grad]": 0.046601254001870984, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data4-4-avg-no_precomputed_grad]": 0.6670472200003132, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data4-4-avg-precomputed_grad]": 0.5045342149987846, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data4-4-full-no_precomputed_grad]": 0.2970590019995143, - "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data4-4-full-precomputed_grad]": 0.3075855399983993, - "tests/influence/torch/test_functional.py::test_hvp[model_data0-1e-05]": 0.021658439001839724, - "tests/influence/torch/test_functional.py::test_hvp[model_data1-1e-05]": 0.059265748001053, - "tests/influence/torch/test_functional.py::test_hvp[model_data2-1e-05]": 0.01743878900015261, - "tests/influence/torch/test_functional.py::test_hvp[model_data3-1e-05]": 0.02271863699934329, - "tests/influence/torch/test_functional.py::test_hvp[model_data4-1e-05]": 0.03280089899999439, - "tests/influence/torch/test_functional.py::test_matrix_jacobian_product[100-5-110]": 0.022210784001799766, - "tests/influence/torch/test_functional.py::test_matrix_jacobian_product[25-10-500]": 0.020637996000004932, - "tests/influence/torch/test_functional.py::test_matrix_jacobian_product[46-1-632]": 0.01833421299852489, - "tests/influence/torch/test_functional.py::test_matrix_jacobian_product[50-3-120]": 0.01751421799781383, - "tests/influence/torch/test_functional.py::test_mixed_derivatives[100-5-512]": 1.7785364399987884, - "tests/influence/torch/test_functional.py::test_mixed_derivatives[25-10-734]": 0.27127136799936125, - "tests/influence/torch/test_functional.py::test_mixed_derivatives[46-1-1000]": 0.18510219099880487, - "tests/influence/torch/test_functional.py::test_mixed_derivatives[50-3-100]": 0.055743695000273874, - "tests/influence/torch/test_functional.py::test_per_sample_gradient[100-5-120]": 0.022160912998515414, - "tests/influence/torch/test_functional.py::test_per_sample_gradient[25-10-550]": 0.0378074430009292, - "tests/influence/torch/test_functional.py::test_per_sample_gradient[46-6-632]": 0.033401361002688645, - "tests/influence/torch/test_functional.py::test_per_sample_gradient[50-3-120]": 0.02004740899974422, - "tests/influence/torch/test_influence_model.py::test_influence_linear_model[cg-train_set_size_200-perturbation]": 4.1002855009992345, - "tests/influence/torch/test_influence_model.py::test_influence_linear_model[cg-train_set_size_200-up]": 4.092100218998894, - "tests/influence/torch/test_influence_model.py::test_influence_linear_model[direct-train_set_size_200-perturbation]": 0.5856196849999833, - "tests/influence/torch/test_influence_model.py::test_influence_linear_model[direct-train_set_size_200-up]": 0.1793102950014145, - "tests/influence/torch/test_influence_model.py::test_influence_linear_model[lissa-train_set_size_200-perturbation]": 74.52067036900007, - "tests/influence/torch/test_influence_model.py::test_influence_linear_model[lissa-train_set_size_200-up]": 72.80234433299847, + "tests/influence/torch/test_functional.py::test_get_hessian[model_data0-4-avg]": 0.09813399301492609, + "tests/influence/torch/test_functional.py::test_get_hessian[model_data0-4-full]": 0.043379645998356864, + "tests/influence/torch/test_functional.py::test_get_hessian[model_data1-5-avg]": 0.3009787189948838, + "tests/influence/torch/test_functional.py::test_get_hessian[model_data1-5-full]": 0.09857722400920466, + "tests/influence/torch/test_functional.py::test_get_hessian[model_data2-10-avg]": 0.06352706399047747, + "tests/influence/torch/test_functional.py::test_get_hessian[model_data2-10-full]": 0.02785925197531469, + "tests/influence/torch/test_functional.py::test_get_hessian[model_data3-8-avg]": 0.12275354197481647, + "tests/influence/torch/test_functional.py::test_get_hessian[model_data3-8-full]": 0.038030802010325715, + "tests/influence/torch/test_functional.py::test_get_hessian[model_data4-4-avg]": 1.568861082982039, + "tests/influence/torch/test_functional.py::test_get_hessian[model_data4-4-full]": 0.3527732029906474, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data0-4-avg-no_precomputed_grad]": 0.029053338977973908, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data0-4-avg-precomputed_grad]": 0.024562739999964833, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data0-4-full-no_precomputed_grad]": 0.02318769003613852, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data0-4-full-precomputed_grad]": 0.02072873100405559, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data1-5-avg-no_precomputed_grad]": 0.10399829200468957, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data1-5-avg-precomputed_grad]": 0.08527540299110115, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data1-5-full-no_precomputed_grad]": 0.0487626249960158, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data1-5-full-precomputed_grad]": 0.06472014603787102, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data2-10-avg-no_precomputed_grad]": 0.021364449989050627, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data2-10-avg-precomputed_grad]": 0.020947107987012714, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data2-10-full-no_precomputed_grad]": 0.01564259297447279, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data2-10-full-precomputed_grad]": 0.018716127960942686, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data3-8-avg-no_precomputed_grad]": 0.030198641994502395, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data3-8-avg-precomputed_grad]": 0.02652753097936511, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data3-8-full-no_precomputed_grad]": 0.02417866201722063, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data3-8-full-precomputed_grad]": 0.020271165965823457, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data4-4-avg-no_precomputed_grad]": 0.295251544972416, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data4-4-avg-precomputed_grad]": 0.24220332800177857, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data4-4-full-no_precomputed_grad]": 0.15127054302138276, + "tests/influence/torch/test_functional.py::test_get_hvp_function[model_data4-4-full-precomputed_grad]": 0.12759025098057464, + "tests/influence/torch/test_functional.py::test_hvp[model_data0-1e-05]": 0.01550043502356857, + "tests/influence/torch/test_functional.py::test_hvp[model_data1-1e-05]": 0.03742475199396722, + "tests/influence/torch/test_functional.py::test_hvp[model_data2-1e-05]": 0.008176781033398584, + "tests/influence/torch/test_functional.py::test_hvp[model_data3-1e-05]": 0.010425448999740183, + "tests/influence/torch/test_functional.py::test_hvp[model_data4-1e-05]": 0.015775886015035212, + "tests/influence/torch/test_functional.py::test_matrix_jacobian_product[100-5-110]": 0.008168531028786674, + "tests/influence/torch/test_functional.py::test_matrix_jacobian_product[25-10-500]": 0.013726653007324785, + "tests/influence/torch/test_functional.py::test_matrix_jacobian_product[46-1-632]": 0.008376071986276656, + "tests/influence/torch/test_functional.py::test_matrix_jacobian_product[50-3-120]": 0.0070768119767308235, + "tests/influence/torch/test_functional.py::test_mixed_derivatives[100-5-512]": 2.475869121000869, + "tests/influence/torch/test_functional.py::test_mixed_derivatives[25-10-734]": 0.19724710599984974, + "tests/influence/torch/test_functional.py::test_mixed_derivatives[46-1-1000]": 0.21653579399571754, + "tests/influence/torch/test_functional.py::test_mixed_derivatives[50-3-100]": 0.038392401998862624, + "tests/influence/torch/test_functional.py::test_per_sample_gradient[100-5-120]": 0.008947794995037839, + "tests/influence/torch/test_functional.py::test_per_sample_gradient[25-10-550]": 0.01637960397056304, + "tests/influence/torch/test_functional.py::test_per_sample_gradient[46-6-632]": 0.016884273994946852, + "tests/influence/torch/test_functional.py::test_per_sample_gradient[50-3-120]": 0.007916716014733538, + "tests/influence/torch/test_functional.py::test_randomized_nystroem_approximation[10-5]": 0.005715728009818122, + "tests/influence/torch/test_functional.py::test_randomized_nystroem_approximation[2-1]": 0.023494494991609827, + "tests/influence/torch/test_functional.py::test_randomized_nystroem_approximation[20-20]": 0.009906815976137295, + "tests/influence/torch/test_influence_model.py::test_influence_linear_model[cg-train_set_size_200-perturbation]": 20.91743779098033, + "tests/influence/torch/test_influence_model.py::test_influence_linear_model[cg-train_set_size_200-up]": 21.55937509299838, + "tests/influence/torch/test_influence_model.py::test_influence_linear_model[direct-train_set_size_200-perturbation]": 0.31099940900458023, + "tests/influence/torch/test_influence_model.py::test_influence_linear_model[direct-train_set_size_200-up]": 0.15369610002380796, + "tests/influence/torch/test_influence_model.py::test_influence_linear_model[lissa-train_set_size_200-perturbation]": 34.23643598801573, + "tests/influence/torch/test_influence_model.py::test_influence_linear_model[lissa-train_set_size_200-up]": 30.393810068024322, "tests/influence/torch/test_influence_model.py::test_influences_arnoldi[conv1d_nn_pert]": 2.6213616719978745, "tests/influence/torch/test_influence_model.py::test_influences_arnoldi[conv1d_nn_up]": 2.9271264809995046, "tests/influence/torch/test_influence_model.py::test_influences_arnoldi[conv1d_no_grad_up]": 1.1280039110006328, @@ -211,167 +214,283 @@ "tests/influence/torch/test_influence_model.py::test_influences_arnoldi[simple_nn_class_up]": 3.4398634410008526, "tests/influence/torch/test_influence_model.py::test_influences_arnoldi[simple_nn_pert]": 1.783800326000346, "tests/influence/torch/test_influence_model.py::test_influences_arnoldi[simple_nn_up]": 1.5235134640006436, - "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv1d_nn_pert]": 2.470179049998478, - "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv1d_nn_up]": 2.3925959699990926, - "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv1d_no_grad_up]": 0.7791441699991992, - "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv2d_nn_pert]": 1.128924710001229, - "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv2d_nn_up]": 1.256267286998991, - "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv3d_nn_pert]": 1.2739636030000838, - "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv3d_nn_up]": 1.2143029310009297, - "tests/influence/torch/test_influence_model.py::test_influences_ekfac[simple_nn_class_up]": 3.279752685999483, - "tests/influence/torch/test_influence_model.py::test_influences_ekfac[simple_nn_pert]": 0.7043358350001654, - "tests/influence/torch/test_influence_model.py::test_influences_ekfac[simple_nn_up]": 0.7221000240006106, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_nn_pert-cg]": 2.666355408999152, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_nn_pert-lissa]": 3.536100011000599, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_nn_up-cg]": 2.8996486520009057, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_nn_up-lissa]": 3.648799233000318, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_no_grad_up-cg]": 0.846027261000927, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_no_grad_up-lissa]": 1.46926116800023, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv2d_nn_pert-cg]": 1.322623816999112, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv2d_nn_pert-lissa]": 2.4566458920016885, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv2d_nn_up-cg]": 1.3719535260006523, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv2d_nn_up-lissa]": 2.3520564940008626, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv3d_nn_pert-cg]": 1.4215319300001283, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv3d_nn_pert-lissa]": 2.5365598410025996, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv3d_nn_up-cg]": 1.4816708039998048, - "tests/influence/torch/test_influence_model.py::test_influences_nn[conv3d_nn_up-lissa]": 2.420441305001077, - "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_class_up-cg]": 3.5962213779985177, - "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_class_up-lissa]": 4.116930427000625, - "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_pert-cg]": 0.8423471179994522, - "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_pert-lissa]": 1.8240221239993843, - "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_up-cg]": 0.8927097550003964, - "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_up-lissa]": 1.7715864019992296, - "tests/influence/torch/test_util.py::test_align_structure_error[source0-target0]": 0.008872623999195639, - "tests/influence/torch/test_util.py::test_align_structure_error[source1-target1]": 0.008990501999505796, - "tests/influence/torch/test_util.py::test_align_structure_error[source2-unsupported]": 0.008625348000350641, - "tests/influence/torch/test_util.py::test_align_structure_success[source0-target0]": 0.009682354999313247, - "tests/influence/torch/test_util.py::test_align_structure_success[source1-target1]": 0.011076738001065678, - "tests/influence/torch/test_util.py::test_align_structure_success[source2-target2]": 0.008422474998951657, - "tests/influence/torch/test_util.py::test_batch_hvp[model_data0-1e-05]": 0.018433343999276985, - "tests/influence/torch/test_util.py::test_batch_hvp[model_data1-1e-05]": 0.04329261199745815, - "tests/influence/torch/test_util.py::test_batch_hvp[model_data2-1e-05]": 0.021779085998787195, - "tests/influence/torch/test_util.py::test_batch_hvp[model_data3-1e-05]": 0.02447877800113929, - "tests/influence/torch/test_util.py::test_batch_hvp[model_data4-1e-05]": 0.027325978000590112, - "tests/influence/torch/test_util.py::test_get_hvp_function[model_data0-4-avg]": 0.05654373300058069, - "tests/influence/torch/test_util.py::test_get_hvp_function[model_data0-4-full]": 0.048235695001494605, - "tests/influence/torch/test_util.py::test_get_hvp_function[model_data1-5-avg]": 0.10194805900027859, - "tests/influence/torch/test_util.py::test_get_hvp_function[model_data1-5-full]": 0.07934144700084289, - "tests/influence/torch/test_util.py::test_get_hvp_function[model_data2-10-avg]": 0.04320316699704563, - "tests/influence/torch/test_util.py::test_get_hvp_function[model_data2-10-full]": 0.03360124099890527, - "tests/influence/torch/test_util.py::test_get_hvp_function[model_data3-8-avg]": 0.062039004000325804, - "tests/influence/torch/test_util.py::test_get_hvp_function[model_data3-8-full]": 0.039968260998648475, - "tests/influence/torch/test_util.py::test_get_hvp_function[model_data4-4-avg]": 0.5075304600013624, - "tests/influence/torch/test_util.py::test_get_hvp_function[model_data4-4-full]": 0.29033965000235185, - "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data0-4-200-0.0001]": 6.130291282001053, - "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data1-5-70-0.001]": 7.575732932000392, - "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data2-10-50-0.0001]": 5.145587835002516, - "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data3-8-160-1e-05]": 9.05795658399984, - "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data4-4-250-1e-05]": 15.930120687000453, - "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx_exception": 0.010992516999976942, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-30-5]": 0.3716939040004945, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-30-6]": 0.3245709369984979, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-45-5]": 0.4916222280025977, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-45-6]": 0.44272739400003047, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-50-5]": 11.764691698001116, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-50-6]": 6.499053524999908, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-30-5]": 0.3747018210015085, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-30-6]": 0.33021277699845086, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-45-5]": 0.5002059710004687, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-45-6]": 0.46272212300027604, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-50-5]": 0.5805674699986412, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-50-6]": 0.5113370569997642, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-30-5]": 0.44749919500100077, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-30-6]": 0.37910716500118724, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-45-5]": 0.5593350939998345, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-45-6]": 0.47198495200063917, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-50-5]": 0.5512238980008988, - "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-50-6]": 0.5188143759987724, - "tests/test_plugin.py::test_failure": 0.003285975997641799, - "tests/test_plugin.py::test_fixture_call_no_arguments": 0.0033647860000201035, - "tests/test_plugin.py::test_fixture_only[1]": 0.0034970750002685236, - "tests/test_plugin.py::test_fixture_only[2]": 0.0034195480002381373, - "tests/test_plugin.py::test_marker_and_fixture[1]": 0.0034149520015489543, - "tests/test_plugin.py::test_marker_and_fixture[2]": 0.003513548002956668, - "tests/test_plugin.py::test_marker_ignore_exception[0]": 0.003384523999557132, - "tests/test_plugin.py::test_marker_ignore_exception[1]": 0.00331666099918948, - "tests/test_plugin.py::test_marker_ignore_exception[2]": 0.00323147399831214, - "tests/test_plugin.py::test_marker_ignore_exception[3]": 0.0032334910029021557, - "tests/test_plugin.py::test_marker_ignore_exception[4]": 0.003230911001082859, - "tests/test_plugin.py::test_marker_only[0]": 0.006288947000939515, - "tests/test_plugin.py::test_marker_only_with_data_fixture[0]": 0.003481858999293763, - "tests/test_plugin.py::test_marker_only_with_data_fixture[1]": 0.003525184998579789, + "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv1d_nn_pert]": 0.9141263900091872, + "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv1d_nn_up]": 1.004119770048419, + "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv1d_no_grad_up]": 0.38047996698878706, + "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv2d_nn_pert]": 0.48015616298653185, + "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv2d_nn_up]": 0.5073693199956324, + "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv3d_nn_pert]": 0.6013624930055812, + "tests/influence/torch/test_influence_model.py::test_influences_ekfac[conv3d_nn_up]": 0.49367734795669094, + "tests/influence/torch/test_influence_model.py::test_influences_ekfac[simple_nn_class_up]": 1.3860138560121413, + "tests/influence/torch/test_influence_model.py::test_influences_ekfac[simple_nn_pert]": 0.3072592499956954, + "tests/influence/torch/test_influence_model.py::test_influences_ekfac[simple_nn_up]": 0.3164721610082779, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv1d_nn_pert-arnoldi]": 1.0992808279988822, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv1d_nn_pert-nystroem]": 1.040673425042769, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv1d_nn_up-arnoldi]": 1.1172903449914884, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv1d_nn_up-nystroem]": 1.036920167010976, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv1d_no_grad_up-arnoldi]": 0.44857565098209307, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv1d_no_grad_up-nystroem]": 0.3657014589989558, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv2d_nn_pert-arnoldi]": 10.615373000007821, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv2d_nn_pert-nystroem]": 0.6304801060177851, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv2d_nn_up-arnoldi]": 12.847174925991567, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv2d_nn_up-nystroem]": 0.6427060620044358, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv3d_nn_pert-arnoldi]": 3.3492665359808598, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv3d_nn_pert-nystroem]": 0.7446247620100621, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv3d_nn_up-arnoldi]": 1.3008154160052072, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[conv3d_nn_up-nystroem]": 0.659043225023197, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[simple_nn_class_up-arnoldi]": 1.4185076340218075, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[simple_nn_class_up-nystroem]": 1.341759205010021, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[simple_nn_pert-arnoldi]": 1.5481110329856165, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[simple_nn_pert-nystroem]": 0.7679775689903181, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[simple_nn_up-arnoldi]": 0.6033870960236527, + "tests/influence/torch/test_influence_model.py::test_influences_low_rank[simple_nn_up-nystroem]": 0.4260945589921903, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_nn_pert-cg]": 1.2415003680216614, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_nn_pert-lissa]": 1.446426609007176, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_nn_up-cg]": 1.3278332729823887, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_nn_up-lissa]": 1.458945247984957, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_no_grad_up-cg]": 0.48686093898140825, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv1d_no_grad_up-lissa]": 0.5758301080204546, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv2d_nn_pert-cg]": 0.7778827689762693, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv2d_nn_pert-lissa]": 0.9706131120037753, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv2d_nn_up-cg]": 0.820894897042308, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv2d_nn_up-lissa]": 1.0038924609834794, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv3d_nn_pert-cg]": 0.8033803289872594, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv3d_nn_pert-lissa]": 1.055458217015257, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv3d_nn_up-cg]": 0.8625257119711023, + "tests/influence/torch/test_influence_model.py::test_influences_nn[conv3d_nn_up-lissa]": 1.0259976829984225, + "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_class_up-cg]": 3.199327295005787, + "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_class_up-lissa]": 1.6534321949875448, + "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_pert-cg]": 0.49723594496026635, + "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_pert-lissa]": 0.8214586230169516, + "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_up-cg]": 0.5237956949567888, + "tests/influence/torch/test_influence_model.py::test_influences_nn[simple_nn_up-lissa]": 0.8058031850086991, + "tests/influence/torch/test_util.py::test_align_structure_error[source0-target0]": 0.006448083964642137, + "tests/influence/torch/test_util.py::test_align_structure_error[source1-target1]": 0.00412756999139674, + "tests/influence/torch/test_util.py::test_align_structure_error[source2-unsupported]": 0.0034424379700794816, + "tests/influence/torch/test_util.py::test_align_structure_success[source0-target0]": 0.005712676967959851, + "tests/influence/torch/test_util.py::test_align_structure_success[source1-target1]": 0.006509427999844775, + "tests/influence/torch/test_util.py::test_align_structure_success[source2-target2]": 0.004126884043216705, + "tests/influence/torch/test_util.py::test_batch_hvp[model_data0-1e-05]": 0.008645498979603872, + "tests/influence/torch/test_util.py::test_batch_hvp[model_data1-1e-05]": 0.019303103996207938, + "tests/influence/torch/test_util.py::test_batch_hvp[model_data2-1e-05]": 0.009336137009086087, + "tests/influence/torch/test_util.py::test_batch_hvp[model_data3-1e-05]": 0.008507663034833968, + "tests/influence/torch/test_util.py::test_batch_hvp[model_data4-1e-05]": 0.009635148977395147, + "tests/influence/torch/test_util.py::test_get_hvp_function[model_data0-4-avg]": 0.02610064498730935, + "tests/influence/torch/test_util.py::test_get_hvp_function[model_data0-4-full]": 0.02072099200449884, + "tests/influence/torch/test_util.py::test_get_hvp_function[model_data1-5-avg]": 0.038389799010474235, + "tests/influence/torch/test_util.py::test_get_hvp_function[model_data1-5-full]": 0.0304763910244219, + "tests/influence/torch/test_util.py::test_get_hvp_function[model_data2-10-avg]": 0.01742512101191096, + "tests/influence/torch/test_util.py::test_get_hvp_function[model_data2-10-full]": 0.01769776103901677, + "tests/influence/torch/test_util.py::test_get_hvp_function[model_data3-8-avg]": 0.02227302800747566, + "tests/influence/torch/test_util.py::test_get_hvp_function[model_data3-8-full]": 0.018950653000501916, + "tests/influence/torch/test_util.py::test_get_hvp_function[model_data4-4-avg]": 0.20586689305491745, + "tests/influence/torch/test_util.py::test_get_hvp_function[model_data4-4-full]": 0.12533411505864933, + "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data0-4-200-0.0001]": 3.6178578450053465, + "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data1-5-70-0.001]": 3.19126154697733, + "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data2-10-50-0.0001]": 0.3648842849652283, + "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data3-8-160-1e-05]": 5.310517774982145, + "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx[model_data4-4-250-1e-05]": 6.977912158996332, + "tests/influence/torch/test_util.py::test_lanzcos_low_rank_hessian_approx_exception": 0.010342207009671256, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-30-5]": 0.19223833800060675, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-30-6]": 0.1771155720052775, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-45-5]": 0.3583740259928163, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-45-6]": 0.23107186099514365, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-50-5]": 6.3838129109935835, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions0-50-6]": 3.9026464489870705, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-30-5]": 0.20121028998983093, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-30-6]": 0.1806995400111191, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-45-5]": 0.24759877700125799, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-45-6]": 0.22885328001575544, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-50-5]": 0.2722237109846901, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions1-50-6]": 0.252675962052308, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-30-5]": 0.19425044997478835, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-30-6]": 0.17584561900002882, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-45-5]": 0.2497809480119031, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-45-6]": 0.24781898900982924, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-50-5]": 0.26879918097984046, + "tests/influence/torch/test_util.py::test_torch_dataset_to_dask_array[tailing_dimensions2-50-6]": 0.26797316601732746, + "tests/parallel/test_parallel.py::test_chunkification[joblib-data0-3-expected_chunks0]": 0.003192901989677921, + "tests/parallel/test_parallel.py::test_chunkification[joblib-data1-2-expected_chunks1]": 0.003511309012537822, + "tests/parallel/test_parallel.py::test_chunkification[joblib-data2-2-expected_chunks2]": 0.00337875698460266, + "tests/parallel/test_parallel.py::test_chunkification[joblib-data3-3-expected_chunks3]": 0.0032286280184052885, + "tests/parallel/test_parallel.py::test_chunkification[joblib-data4-5-expected_chunks4]": 0.0035956800275016576, + "tests/parallel/test_parallel.py::test_chunkification[joblib-data5-42-expected_chunks5]": 0.003795255965087563, + "tests/parallel/test_parallel.py::test_chunkification[joblib-data6-42-expected_chunks6]": 0.003133207996143028, + "tests/parallel/test_parallel.py::test_chunkification[joblib-data7-4-expected_chunks7]": 0.004749076993903145, + "tests/parallel/test_parallel.py::test_chunkification[joblib-data8-4-expected_chunks8]": 0.0034613849711604416, + "tests/parallel/test_parallel.py::test_chunkification[ray-external-data0-3-expected_chunks0]": 0.0031638060172554106, + "tests/parallel/test_parallel.py::test_chunkification[ray-external-data1-2-expected_chunks1]": 0.004364188993349671, + "tests/parallel/test_parallel.py::test_chunkification[ray-external-data2-2-expected_chunks2]": 0.005876399023691192, + "tests/parallel/test_parallel.py::test_chunkification[ray-external-data3-3-expected_chunks3]": 0.0034405409824103117, + "tests/parallel/test_parallel.py::test_chunkification[ray-external-data4-5-expected_chunks4]": 0.0038114589697215706, + "tests/parallel/test_parallel.py::test_chunkification[ray-external-data5-42-expected_chunks5]": 0.003221377992304042, + "tests/parallel/test_parallel.py::test_chunkification[ray-external-data6-42-expected_chunks6]": 0.0032008349953684956, + "tests/parallel/test_parallel.py::test_chunkification[ray-external-data7-4-expected_chunks7]": 0.003124484996078536, + "tests/parallel/test_parallel.py::test_chunkification[ray-external-data8-4-expected_chunks8]": 0.0033526600163895637, + "tests/parallel/test_parallel.py::test_chunkification[ray-local-data0-3-expected_chunks0]": 0.006123727012891322, + "tests/parallel/test_parallel.py::test_chunkification[ray-local-data1-2-expected_chunks1]": 0.0053288199997041374, + "tests/parallel/test_parallel.py::test_chunkification[ray-local-data2-2-expected_chunks2]": 0.005889113061130047, + "tests/parallel/test_parallel.py::test_chunkification[ray-local-data3-3-expected_chunks3]": 0.006710874993586913, + "tests/parallel/test_parallel.py::test_chunkification[ray-local-data4-5-expected_chunks4]": 0.00494873701245524, + "tests/parallel/test_parallel.py::test_chunkification[ray-local-data5-42-expected_chunks5]": 0.005258859979221597, + "tests/parallel/test_parallel.py::test_chunkification[ray-local-data6-42-expected_chunks6]": 0.0059889889671467245, + "tests/parallel/test_parallel.py::test_chunkification[ray-local-data7-4-expected_chunks7]": 0.00598992602317594, + "tests/parallel/test_parallel.py::test_chunkification[ray-local-data8-4-expected_chunks8]": 0.004992969945305958, + "tests/parallel/test_parallel.py::test_effective_n_jobs[joblib]": 0.0040679979720152915, + "tests/parallel/test_parallel.py::test_effective_n_jobs[ray-external]": 3.7438874400395434, + "tests/parallel/test_parallel.py::test_effective_n_jobs[ray-local]": 4.747794731985778, + "tests/parallel/test_parallel.py::test_future_cancellation[joblib]": 0.0032916720083449036, + "tests/parallel/test_parallel.py::test_future_cancellation[ray-external]": 6.0036520949797705, + "tests/parallel/test_parallel.py::test_future_cancellation[ray-local]": 5.18983832403319, + "tests/parallel/test_parallel.py::test_futures_executor_map[joblib]": 1.036263770016376, + "tests/parallel/test_parallel.py::test_futures_executor_map[ray-external]": 0.09937145400908776, + "tests/parallel/test_parallel.py::test_futures_executor_map[ray-local]": 0.1012835220026318, + "tests/parallel/test_parallel.py::test_futures_executor_map_with_max_workers[joblib]": 0.005493839009432122, + "tests/parallel/test_parallel.py::test_futures_executor_map_with_max_workers[ray-external]": 1.0856830689881463, + "tests/parallel/test_parallel.py::test_futures_executor_map_with_max_workers[ray-local]": 1.1039586100087035, + "tests/parallel/test_parallel.py::test_futures_executor_submit[joblib]": 1.0226233069843147, + "tests/parallel/test_parallel.py::test_futures_executor_submit[ray-external]": 0.04702838300727308, + "tests/parallel/test_parallel.py::test_futures_executor_submit[ray-local]": 0.05678758802241646, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-1-list-indices0-expected0]": 0.00467701000161469, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-1-list-indices1-expected1]": 0.0042948100017383695, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-1-list-indices2-expected2]": 0.0030402019910980016, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-1-numpy-indices4-45]": 0.003510809998260811, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-1-range-indices3-expected3]": 0.0033703809895087034, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-2-list-indices0-expected0]": 0.0041150090110022575, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-2-list-indices1-expected1]": 0.004162732977420092, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-2-list-indices2-expected2]": 0.003259133023675531, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-2-numpy-indices4-45]": 0.004144402977544814, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-2-range-indices3-expected3]": 0.004427063016919419, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-4-list-indices0-expected0]": 0.0029944690177217126, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-4-list-indices1-expected1]": 0.003702607995364815, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-4-list-indices2-expected2]": 0.0037922649644315243, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-4-numpy-indices4-45]": 0.003974005987402052, + "tests/parallel/test_parallel.py::test_map_reduce_job[joblib-4-range-indices3-expected3]": 0.003567876963643357, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-1-list-indices0-expected0]": 0.005004247970646247, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-1-list-indices1-expected1]": 0.004017235012724996, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-1-list-indices2-expected2]": 0.004506341996602714, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-1-numpy-indices4-45]": 0.003820352052571252, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-1-range-indices3-expected3]": 0.004570545977912843, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-2-list-indices0-expected0]": 0.004423253994900733, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-2-list-indices1-expected1]": 0.006525125994812697, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-2-list-indices2-expected2]": 0.004407291999086738, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-2-numpy-indices4-45]": 0.003792414005147293, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-2-range-indices3-expected3]": 0.0040772780193947256, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-4-list-indices0-expected0]": 0.003901798016158864, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-4-list-indices1-expected1]": 0.003945255011785775, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-4-list-indices2-expected2]": 0.003970507998019457, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-4-numpy-indices4-45]": 0.003773231990635395, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-external-4-range-indices3-expected3]": 0.0037270110042300075, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-1-list-indices0-expected0]": 0.014644921000581235, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-1-list-indices1-expected1]": 0.016928659984841943, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-1-list-indices2-expected2]": 0.010305310046533123, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-1-numpy-indices4-45]": 0.008892301004379988, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-1-range-indices3-expected3]": 0.014073127938900143, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-2-list-indices0-expected0]": 0.007267932000104338, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-2-list-indices1-expected1]": 0.007973060040967539, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-2-list-indices2-expected2]": 0.008930301031796262, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-2-numpy-indices4-45]": 0.009722396003780887, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-2-range-indices3-expected3]": 0.00856516498606652, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-4-list-indices0-expected0]": 0.013356091978494078, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-4-list-indices1-expected1]": 0.006517391972010955, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-4-list-indices2-expected2]": 0.010245340003166348, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-4-numpy-indices4-45]": 0.006771289015887305, + "tests/parallel/test_parallel.py::test_map_reduce_job[ray-local-4-range-indices3-expected3]": 0.007704173040110618, + "tests/parallel/test_parallel.py::test_map_reduce_job_partial_map_and_reduce_func[joblib]": 0.004209100996376947, + "tests/parallel/test_parallel.py::test_map_reduce_job_partial_map_and_reduce_func[ray-external]": 0.002531879028538242, + "tests/parallel/test_parallel.py::test_map_reduce_job_partial_map_and_reduce_func[ray-local]": 0.006263459014007822, + "tests/parallel/test_parallel.py::test_map_reduce_seeding[joblib-42-12]": 0.015576867037452757, + "tests/parallel/test_parallel.py::test_map_reduce_seeding[ray-external-42-12]": 0.005515481985639781, + "tests/parallel/test_parallel.py::test_map_reduce_seeding[ray-local-42-12]": 0.008114579977700487, + "tests/parallel/test_parallel.py::test_wrap_function[joblib]": 0.0033907410106621683, + "tests/parallel/test_parallel.py::test_wrap_function[ray-external]": 2.7244514969934244, + "tests/parallel/test_parallel.py::test_wrap_function[ray-local]": 3.2604004809982143, + "tests/test_plugin.py::test_failure": 0.0017938569653779268, + "tests/test_plugin.py::test_fixture_call_no_arguments": 0.0019022209744434804, + "tests/test_plugin.py::test_fixture_only[1]": 0.002128774009179324, + "tests/test_plugin.py::test_fixture_only[2]": 0.002033632976235822, + "tests/test_plugin.py::test_marker_and_fixture[1]": 0.0019118520140182227, + "tests/test_plugin.py::test_marker_and_fixture[2]": 0.001956833031727001, + "tests/test_plugin.py::test_marker_ignore_exception[0]": 0.0017019310325849801, + "tests/test_plugin.py::test_marker_ignore_exception[1]": 0.0017486069991718978, + "tests/test_plugin.py::test_marker_ignore_exception[2]": 0.0016994159668684006, + "tests/test_plugin.py::test_marker_ignore_exception[3]": 0.0017209959914907813, + "tests/test_plugin.py::test_marker_ignore_exception[4]": 0.0017584379820618778, + "tests/test_plugin.py::test_marker_only[0]": 0.004290158016374335, + "tests/test_plugin.py::test_marker_only_with_data_fixture[0]": 0.0020319949835538864, + "tests/test_plugin.py::test_marker_only_with_data_fixture[1]": 0.0022678270179312676, "tests/test_plugin.py::test_marker_only_with_data_fixture[2]": 0.0012167239910922945, - "tests/test_results.py::test_adding_different_indices[indices_10-names_10-values_10-indices_20-names_20-values_20-expected_indices0-expected_names0-expected_values0]": 0.005255071999272332, - "tests/test_results.py::test_adding_different_indices[indices_11-names_11-values_11-indices_21-names_21-values_21-expected_indices1-expected_names1-expected_values1]": 0.005423882998002227, - "tests/test_results.py::test_adding_different_indices[indices_12-names_12-values_12-indices_22-names_22-values_22-expected_indices2-expected_names2-expected_values2]": 0.0054176030007511145, - "tests/test_results.py::test_adding_different_indices[indices_13-names_13-values_13-indices_23-names_23-values_23-expected_indices3-expected_names3-expected_values3]": 0.005320759000824182, - "tests/test_results.py::test_adding_random": 0.008090201999948476, - "tests/test_results.py::test_dataframe_sorting[values0-names0-ranks_asc0]": 0.008680627001012908, - "tests/test_results.py::test_dataframe_sorting[values1-names1-ranks_asc1]": 0.006279957000515424, - "tests/test_results.py::test_empty[0]": 0.003641559000243433, - "tests/test_results.py::test_empty[5]": 0.0034929009998450056, + "tests/test_results.py::test_adding_different_indices[indices_10-names_10-values_10-indices_20-names_20-values_20-expected_indices0-expected_names0-expected_values0]": 0.0028672630432993174, + "tests/test_results.py::test_adding_different_indices[indices_11-names_11-values_11-indices_21-names_21-values_21-expected_indices1-expected_names1-expected_values1]": 0.002818326000124216, + "tests/test_results.py::test_adding_different_indices[indices_12-names_12-values_12-indices_22-names_22-values_22-expected_indices2-expected_names2-expected_values2]": 0.0028436680149752647, + "tests/test_results.py::test_adding_different_indices[indices_13-names_13-values_13-indices_23-names_23-values_23-expected_indices3-expected_names3-expected_values3]": 0.0028777279949281365, + "tests/test_results.py::test_adding_random": 0.004541941947536543, + "tests/test_results.py::test_dataframe_sorting[values0-names0-ranks_asc0]": 0.0067770640016533434, + "tests/test_results.py::test_dataframe_sorting[values1-names1-ranks_asc1]": 0.003599408984882757, + "tests/test_results.py::test_empty[0]": 0.0018440319981891662, + "tests/test_results.py::test_empty[5]": 0.001971584017155692, "tests/test_results.py::test_empty_deprecation": 0.0035087529995507794, - "tests/test_results.py::test_equality[values0-names0]": 0.004204769002171815, - "tests/test_results.py::test_equality[values1-names1]": 0.004355032000603387, - "tests/test_results.py::test_extra_values[extra_values0]": 0.004352300002210541, - "tests/test_results.py::test_extra_values[extra_values1]": 0.004183073002423043, - "tests/test_results.py::test_from_random_creation[-1.0-10]": 0.003700102000948391, - "tests/test_results.py::test_from_random_creation[-1.0-1]": 0.0037578669980575796, - "tests/test_results.py::test_from_random_creation[1.0-10]": 0.0037663590010197368, - "tests/test_results.py::test_from_random_creation[1.0-1]": 0.0038107559994386975, - "tests/test_results.py::test_from_random_creation[None-10]": 0.0036299099992902484, - "tests/test_results.py::test_from_random_creation[None-1]": 0.0037064750013087178, - "tests/test_results.py::test_from_random_creation_errors": 0.00302240999735659, - "tests/test_results.py::test_get_idx": 0.0032395419984823093, - "tests/test_results.py::test_indexing[values0-names0-ranks_asc0]": 0.004395261999889044, - "tests/test_results.py::test_indexing[values1-names1-ranks_asc1]": 0.004297610999856261, - "tests/test_results.py::test_iter[values0-names0-ranks_asc0]": 0.003994380002040998, - "tests/test_results.py::test_iter[values1-names1-ranks_asc1]": 0.004105043000890873, - "tests/test_results.py::test_names[data_names0]": 0.003977623999162461, - "tests/test_results.py::test_serialization[values0-None-dumps-loads0]": 0.004757332000735914, - "tests/test_results.py::test_serialization[values0-None-dumps-loads1]": 0.004781503001140663, - "tests/test_results.py::test_serialization[values1-None-dumps-loads0]": 0.004697013004260953, - "tests/test_results.py::test_serialization[values1-None-dumps-loads1]": 0.004567677997329156, - "tests/test_results.py::test_sorting[values0-names0-ranks_asc0]": 0.004288738000468584, - "tests/test_results.py::test_sorting[values1-names1-ranks_asc1]": 0.00483136099865078, - "tests/test_results.py::test_todataframe[values0-names0-ranks_asc0]": 0.0060278359978838125, - "tests/test_results.py::test_todataframe[values1-names1-ranks_asc1]": 0.006075087001590873, - "tests/test_results.py::test_types[indices0-int32-data_names0---True]": 0.013553835999118746, - "tests/utils/test_caching.py::test_cached_func_hash_function[foo--False]": 0.009608976999516017, - "tests/utils/test_caching.py::test_cached_func_hash_function[foo-foo-True]": 0.007844682999348151, - "tests/utils/test_caching.py::test_cached_func_hash_function[foo-foo_duplicate-True]": 0.00921746999847528, - "tests/utils/test_caching.py::test_cached_func_hash_function[foo-foo_with_random-False]": 0.006780997000532807, - "tests/utils/test_caching.py::test_cached_func_hash_function[foo_with_random-foo_with_random_and_sleep-False]": 0.008397087000048487, + "tests/test_results.py::test_equality[values0-names0]": 0.002252487960504368, + "tests/test_results.py::test_equality[values1-names1]": 0.002345187996979803, + "tests/test_results.py::test_extra_values[extra_values0]": 0.002402268990408629, + "tests/test_results.py::test_extra_values[extra_values1]": 0.0022489750117529184, + "tests/test_results.py::test_from_random_creation[-1.0-10]": 0.0019749769708141685, + "tests/test_results.py::test_from_random_creation[-1.0-1]": 0.0020128499891143292, + "tests/test_results.py::test_from_random_creation[1.0-10]": 0.002059541002381593, + "tests/test_results.py::test_from_random_creation[1.0-1]": 0.00199978199088946, + "tests/test_results.py::test_from_random_creation[None-10]": 0.0019230550387874246, + "tests/test_results.py::test_from_random_creation[None-1]": 0.0020171999640297145, + "tests/test_results.py::test_from_random_creation_errors": 0.0015933870163280517, + "tests/test_results.py::test_get_idx": 0.0019429969834163785, + "tests/test_results.py::test_indexing[values0-names0-ranks_asc0]": 0.0025470299879089, + "tests/test_results.py::test_indexing[values1-names1-ranks_asc1]": 0.0024958310241345316, + "tests/test_results.py::test_iter[values0-names0-ranks_asc0]": 0.002205308002885431, + "tests/test_results.py::test_iter[values1-names1-ranks_asc1]": 0.0023963370185811073, + "tests/test_results.py::test_names[data_names0]": 0.0021665350068360567, + "tests/test_results.py::test_serialization[values0-None-dumps-loads0]": 0.0026234709948766977, + "tests/test_results.py::test_serialization[values0-None-dumps-loads1]": 0.002808505989378318, + "tests/test_results.py::test_serialization[values1-None-dumps-loads0]": 0.0025294710067100823, + "tests/test_results.py::test_serialization[values1-None-dumps-loads1]": 0.0024129379889927804, + "tests/test_results.py::test_sorting[values0-names0-ranks_asc0]": 0.0021807790035381913, + "tests/test_results.py::test_sorting[values1-names1-ranks_asc1]": 0.00987413403345272, + "tests/test_results.py::test_todataframe[values0-names0-ranks_asc0]": 0.0034775169915519655, + "tests/test_results.py::test_todataframe[values1-names1-ranks_asc1]": 0.003345073026139289, + "tests/test_results.py::test_types[indices0-int32-data_names0---True]": 0.007417112967232242, + "tests/utils/test_caching.py::test_cached_func_hash_function[foo--False]": 0.004049851006129757, + "tests/utils/test_caching.py::test_cached_func_hash_function[foo-foo-True]": 0.002690388966584578, + "tests/utils/test_caching.py::test_cached_func_hash_function[foo-foo_duplicate-True]": 0.002542914997320622, + "tests/utils/test_caching.py::test_cached_func_hash_function[foo-foo_with_random-False]": 0.0028115719615016133, + "tests/utils/test_caching.py::test_cached_func_hash_function[foo_with_random-foo_with_random_and_sleep-False]": 0.0031461880425922573, "tests/utils/test_caching.py::test_failed_connection": 0.0039788429858163, - "tests/utils/test_caching.py::test_faster_with_repeated_training[disk]": 5.499508081999011, - "tests/utils/test_caching.py::test_faster_with_repeated_training[in-memory]": 5.596929604998877, - "tests/utils/test_caching.py::test_faster_with_repeated_training[memcached]": 6.545152930997574, - "tests/utils/test_caching.py::test_memcached_failed_connection": 0.009630470000047353, + "tests/utils/test_caching.py::test_faster_with_repeated_training[disk]": 5.462864195025759, + "tests/utils/test_caching.py::test_faster_with_repeated_training[in-memory]": 5.410583495016908, + "tests/utils/test_caching.py::test_faster_with_repeated_training[memcached]": 5.505492224008776, + "tests/utils/test_caching.py::test_memcached_failed_connection": 0.0027141640021000057, "tests/utils/test_caching.py::test_memcached_faster_with_repeated_training": 5.003239913989091, "tests/utils/test_caching.py::test_memcached_parallel_jobs[joblib]": 3.1677759810409043, "tests/utils/test_caching.py::test_memcached_parallel_jobs[ray-external]": 38.430890925985295, @@ -390,15 +509,26 @@ "tests/utils/test_caching.py::test_memcached_parallel_repeated_training[ray-local-20-2-20-10]": 0.007027510990155861, "tests/utils/test_caching.py::test_memcached_repeated_training": 2.3077823049970903, "tests/utils/test_caching.py::test_memcached_single_job": 0.007132280006771907, + "tests/utils/test_caching.py::test_parallel_jobs[disk]": 0.0027724569954443723, + "tests/utils/test_caching.py::test_parallel_jobs[in-memory]": 0.0026654380199033767, "tests/utils/test_caching.py::test_parallel_jobs[joblib-disk]": 0.008082594998995773, "tests/utils/test_caching.py::test_parallel_jobs[joblib-in-memory]": 0.007858986000428558, "tests/utils/test_caching.py::test_parallel_jobs[joblib-memcached]": 5.864486223999847, + "tests/utils/test_caching.py::test_parallel_jobs[memcached]": 0.023942797997733578, "tests/utils/test_caching.py::test_parallel_jobs[ray-external-disk]": 0.019337756999448175, "tests/utils/test_caching.py::test_parallel_jobs[ray-external-in-memory]": 3.8737009590004163, "tests/utils/test_caching.py::test_parallel_jobs[ray-external-memcached]": 0.010424148002130096, "tests/utils/test_caching.py::test_parallel_jobs[ray-local-disk]": 0.006320855998637853, "tests/utils/test_caching.py::test_parallel_jobs[ray-local-in-memory]": 0.007159704999139649, "tests/utils/test_caching.py::test_parallel_jobs[ray-local-memcached]": 0.010268650999933016, + "tests/utils/test_caching.py::test_parallel_repeated_training[disk-20-1-10-5]": 0.0175380950095132, + "tests/utils/test_caching.py::test_parallel_repeated_training[disk-20-1-20-10]": 0.013737032044446096, + "tests/utils/test_caching.py::test_parallel_repeated_training[disk-20-2-10-5]": 0.023939374019391835, + "tests/utils/test_caching.py::test_parallel_repeated_training[disk-20-2-20-10]": 0.02181547399959527, + "tests/utils/test_caching.py::test_parallel_repeated_training[in-memory-20-1-10-5]": 0.015921188984066248, + "tests/utils/test_caching.py::test_parallel_repeated_training[in-memory-20-1-20-10]": 0.011334646958857775, + "tests/utils/test_caching.py::test_parallel_repeated_training[in-memory-20-2-10-5]": 0.017839537031250075, + "tests/utils/test_caching.py::test_parallel_repeated_training[in-memory-20-2-20-10]": 0.01734064202173613, "tests/utils/test_caching.py::test_parallel_repeated_training[joblib-disk-20-1-10-5]": 0.040544517996750074, "tests/utils/test_caching.py::test_parallel_repeated_training[joblib-disk-20-1-20-10]": 0.041609834999690065, "tests/utils/test_caching.py::test_parallel_repeated_training[joblib-disk-20-2-10-5]": 0.450297680001313, @@ -411,6 +541,10 @@ "tests/utils/test_caching.py::test_parallel_repeated_training[joblib-memcached-20-1-20-10]": 0.04425754300064, "tests/utils/test_caching.py::test_parallel_repeated_training[joblib-memcached-20-2-10-5]": 0.46746473100029107, "tests/utils/test_caching.py::test_parallel_repeated_training[joblib-memcached-20-2-20-10]": 0.47426626100059366, + "tests/utils/test_caching.py::test_parallel_repeated_training[memcached-20-1-10-5]": 0.016388149961130694, + "tests/utils/test_caching.py::test_parallel_repeated_training[memcached-20-1-20-10]": 0.01673590997233987, + "tests/utils/test_caching.py::test_parallel_repeated_training[memcached-20-2-10-5]": 0.027201458025956526, + "tests/utils/test_caching.py::test_parallel_repeated_training[memcached-20-2-20-10]": 0.027168086962774396, "tests/utils/test_caching.py::test_parallel_repeated_training[ray-external-disk-20-1-10-5]": 0.019769640000959043, "tests/utils/test_caching.py::test_parallel_repeated_training[ray-external-disk-20-1-20-10]": 0.02465987799951108, "tests/utils/test_caching.py::test_parallel_repeated_training[ray-external-disk-20-2-10-5]": 0.012952293998750974, @@ -435,75 +569,75 @@ "tests/utils/test_caching.py::test_parallel_repeated_training[ray-local-memcached-20-1-20-10]": 0.010128158999577863, "tests/utils/test_caching.py::test_parallel_repeated_training[ray-local-memcached-20-2-10-5]": 0.013161438002498471, "tests/utils/test_caching.py::test_parallel_repeated_training[ray-local-memcached-20-2-20-10]": 0.009240641998985666, - "tests/utils/test_caching.py::test_repeated_training[disk]": 1.2679626049975923, - "tests/utils/test_caching.py::test_repeated_training[in-memory]": 0.8953080740011501, - "tests/utils/test_caching.py::test_repeated_training[memcached]": 1.0694843190012762, - "tests/utils/test_caching.py::test_single_job[disk]": 0.013467190003211726, - "tests/utils/test_caching.py::test_single_job[in-memory]": 0.007606943001519539, - "tests/utils/test_caching.py::test_single_job[memcached]": 0.012050191000525956, - "tests/utils/test_caching.py::test_without_pymemcache": 0.0068226680014049634, - "tests/utils/test_dataset.py::test_creating_dataset_from_sklearn[0.1]": 0.020587041000908357, - "tests/utils/test_dataset.py::test_creating_dataset_from_sklearn[0.5]": 0.00390724699900602, - "tests/utils/test_dataset.py::test_creating_dataset_from_sklearn[0.8]": 0.00450960899979691, - "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.1-kwargs0]": 0.0038753029984945897, - "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.1-kwargs1]": 0.0036959019998903386, - "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.5-kwargs0]": 0.0038168650007719407, - "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.5-kwargs1]": 0.0037348340010794345, - "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.8-kwargs0]": 0.003279165001004003, - "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.8-kwargs1]": 0.003160262998790131, - "tests/utils/test_dataset.py::test_creating_dataset_subsclassfrom_sklearn[0.1]": 0.00434540000060224, - "tests/utils/test_dataset.py::test_creating_dataset_subsclassfrom_sklearn[0.5]": 0.004031194001072436, - "tests/utils/test_dataset.py::test_creating_dataset_subsclassfrom_sklearn[0.8]": 0.0037131489989405964, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn[0.1]": 0.0038164179986779345, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn[0.5]": 0.005253569997876184, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn[0.8]": 0.005844020997756161, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn_failure[0.1]": 0.003921200999684515, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn_failure[0.5]": 0.0038101809986983426, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn_failure[0.8]": 0.00419950299874472, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.1-kwargs0]": 0.0037465159985003993, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.1-kwargs1]": 0.0037122550002095522, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.5-kwargs0]": 0.003727491999597987, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.5-kwargs1]": 0.00471100999857299, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.8-kwargs0]": 0.004182996997769806, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.8-kwargs1]": 0.004742823000924545, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays_failure[0.1]": 0.0034745570010272786, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays_failure[0.5]": 0.0029412200001388555, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays_failure[0.8]": 0.003702544998304802, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_subsclassfrom_sklearn[0.1]": 0.004099161000340246, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_subsclassfrom_sklearn[0.5]": 0.004149230999246356, - "tests/utils/test_dataset.py::test_creating_grouped_dataset_subsclassfrom_sklearn[0.8]": 0.004327725000621285, - "tests/utils/test_dataset.py::test_grouped_dataset_results": 0.005107523998958641, - "tests/utils/test_numeric.py::test_powerset": 0.003924966000340646, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[0-2-ValueError]": 0.003189409999322379, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[1-2-ValueError]": 0.0027116169985674787, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[10-1-ValueError]": 0.0027666500009218, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[10-2-None]": 0.0031559840026602615, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[2-10-None]": 0.004578909001793363, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[4--2-ValueError]": 0.0026738769993244205, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[7-23-None]": 0.0031755019990669098, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_reproducible[10-2]": 0.0032696249982109293, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_reproducible[2-10]": 0.0026570699992589653, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_reproducible[7-23]": 0.004022232000352233, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_stochastic[10-2]": 0.004431671000929782, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_stochastic[2-10]": 0.003217298000890878, - "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_stochastic[7-23]": 0.002955772999484907, - "tests/utils/test_numeric.py::test_random_powerset[0-1]": 0.003080959002545569, - "tests/utils/test_numeric.py::test_random_powerset[1-10]": 0.0025249310019717086, - "tests/utils/test_numeric.py::test_random_powerset[10-1024]": 0.012735986998450244, - "tests/utils/test_numeric.py::test_random_powerset[5-128]": 0.0035692419987753965, - "tests/utils/test_numeric.py::test_random_powerset_label_min[0-10-3-1000]": 0.171625541001049, - "tests/utils/test_numeric.py::test_random_powerset_label_min[1-10-3-1000]": 0.17619158500019694, - "tests/utils/test_numeric.py::test_random_powerset_label_min[2-10-3-1000]": 0.1760632390014507, - "tests/utils/test_numeric.py::test_random_powerset_reproducible[10-1024]": 0.018217550998087972, - "tests/utils/test_numeric.py::test_random_powerset_stochastic[10-1024]": 0.018896675997893908, - "tests/utils/test_numeric.py::test_random_subset_of_size[0-0-None]": 0.002780025000902242, - "tests/utils/test_numeric.py::test_random_subset_of_size[0-1-ValueError]": 0.0032452249997731997, - "tests/utils/test_numeric.py::test_random_subset_of_size[10-0-None]": 0.0033025680004357127, - "tests/utils/test_numeric.py::test_random_subset_of_size[10-3-None]": 0.002845983000952401, - "tests/utils/test_numeric.py::test_random_subset_of_size[1000-40-None]": 0.0032918939996307017, - "tests/utils/test_numeric.py::test_random_subset_of_size_stochastic[10-3]": 0.002797532002659864, - "tests/utils/test_numeric.py::test_random_subset_of_size_stochastic[1000-40]": 0.0036268280000513187, - "tests/utils/test_numeric.py::test_running_moments": 0.6145333489985205, + "tests/utils/test_caching.py::test_repeated_training[disk]": 0.33966069895541295, + "tests/utils/test_caching.py::test_repeated_training[in-memory]": 0.23885704105487093, + "tests/utils/test_caching.py::test_repeated_training[memcached]": 0.36306702499859966, + "tests/utils/test_caching.py::test_single_job[disk]": 0.0037449339870363474, + "tests/utils/test_caching.py::test_single_job[in-memory]": 0.0029351940320339054, + "tests/utils/test_caching.py::test_single_job[memcached]": 0.005145985021954402, + "tests/utils/test_caching.py::test_without_pymemcache": 0.005478426028275862, + "tests/utils/test_dataset.py::test_creating_dataset_from_sklearn[0.1]": 0.008665675995871425, + "tests/utils/test_dataset.py::test_creating_dataset_from_sklearn[0.5]": 0.00487168098334223, + "tests/utils/test_dataset.py::test_creating_dataset_from_sklearn[0.8]": 0.005765877984231338, + "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.1-kwargs0]": 0.004334063007263467, + "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.1-kwargs1]": 0.003661135968286544, + "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.5-kwargs0]": 0.0034260130196344107, + "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.5-kwargs1]": 0.0037207130226306617, + "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.8-kwargs0]": 0.0036306940019130707, + "tests/utils/test_dataset.py::test_creating_dataset_from_x_y_arrays[0.8-kwargs1]": 0.0032633960363455117, + "tests/utils/test_dataset.py::test_creating_dataset_subsclassfrom_sklearn[0.1]": 0.003974422957981005, + "tests/utils/test_dataset.py::test_creating_dataset_subsclassfrom_sklearn[0.5]": 0.004185625031823292, + "tests/utils/test_dataset.py::test_creating_dataset_subsclassfrom_sklearn[0.8]": 0.003982000023825094, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn[0.1]": 0.005084333999548107, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn[0.5]": 0.0051782039809040725, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn[0.8]": 0.00535002100514248, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn_failure[0.1]": 0.003540790028637275, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn_failure[0.5]": 0.004949523019604385, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_sklearn_failure[0.8]": 0.0035746630164794624, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.1-kwargs0]": 0.004036521015223116, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.1-kwargs1]": 0.0036350690061226487, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.5-kwargs0]": 0.004778611968504265, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.5-kwargs1]": 0.004359662008937448, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.8-kwargs0]": 0.004105383006390184, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays[0.8-kwargs1]": 0.003795194992562756, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays_failure[0.1]": 0.002760191011475399, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays_failure[0.5]": 0.0043617659830488265, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_from_x_y_arrays_failure[0.8]": 0.003878229996189475, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_subsclassfrom_sklearn[0.1]": 0.005001434998121113, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_subsclassfrom_sklearn[0.5]": 0.004866438015596941, + "tests/utils/test_dataset.py::test_creating_grouped_dataset_subsclassfrom_sklearn[0.8]": 0.005906352016609162, + "tests/utils/test_dataset.py::test_grouped_dataset_results": 0.013181563990656286, + "tests/utils/test_numeric.py::test_powerset": 0.004835293017094955, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[0-2-ValueError]": 0.0027691390132531524, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[1-2-ValueError]": 0.0026925769925583154, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[10-1-ValueError]": 0.002887230977648869, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[10-2-None]": 0.002661020989762619, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[2-10-None]": 0.006404095009202138, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[4--2-ValueError]": 0.002480320050381124, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number[7-23-None]": 0.002957596007036045, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_reproducible[10-2]": 0.0030029160261619836, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_reproducible[2-10]": 0.0029755540017504245, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_reproducible[7-23]": 0.0025017989974003285, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_stochastic[10-2]": 0.002511145023163408, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_stochastic[2-10]": 0.0028129699639976025, + "tests/utils/test_numeric.py::test_random_matrix_with_condition_number_stochastic[7-23]": 0.0027172630361746997, + "tests/utils/test_numeric.py::test_random_powerset[0-1]": 0.003557410993380472, + "tests/utils/test_numeric.py::test_random_powerset[1-10]": 0.00345834597828798, + "tests/utils/test_numeric.py::test_random_powerset[10-1024]": 0.012248383020050824, + "tests/utils/test_numeric.py::test_random_powerset[5-128]": 0.004129782988457009, + "tests/utils/test_numeric.py::test_random_powerset_label_min[0-10-3-1000]": 0.18313832598505542, + "tests/utils/test_numeric.py::test_random_powerset_label_min[1-10-3-1000]": 0.18961101496824995, + "tests/utils/test_numeric.py::test_random_powerset_label_min[2-10-3-1000]": 0.18344586697639897, + "tests/utils/test_numeric.py::test_random_powerset_reproducible[10-1024]": 0.021032890013884753, + "tests/utils/test_numeric.py::test_random_powerset_stochastic[10-1024]": 0.020289141015382484, + "tests/utils/test_numeric.py::test_random_subset_of_size[0-0-None]": 0.004669596964959055, + "tests/utils/test_numeric.py::test_random_subset_of_size[0-1-ValueError]": 0.002807142009260133, + "tests/utils/test_numeric.py::test_random_subset_of_size[10-0-None]": 0.0025325829919893295, + "tests/utils/test_numeric.py::test_random_subset_of_size[10-3-None]": 0.0026709649828262627, + "tests/utils/test_numeric.py::test_random_subset_of_size[1000-40-None]": 0.0033286060206592083, + "tests/utils/test_numeric.py::test_random_subset_of_size_stochastic[10-3]": 0.0026626450126059353, + "tests/utils/test_numeric.py::test_random_subset_of_size_stochastic[1000-40]": 0.002802837989293039, + "tests/utils/test_numeric.py::test_running_moments": 0.6196783449850045, "tests/utils/test_parallel.py::test_chunkification[joblib-data0-3-expected_chunks0]": 0.015510658000494004, "tests/utils/test_parallel.py::test_chunkification[joblib-data1-2-expected_chunks1]": 0.012093620000086958, "tests/utils/test_parallel.py::test_chunkification[joblib-data2-2-expected_chunks2]": 0.011375399999451474, @@ -600,229 +734,229 @@ "tests/utils/test_parallel.py::test_wrap_function[joblib]": 0.010273419000441208, "tests/utils/test_parallel.py::test_wrap_function[ray-external]": 4.3178896000026725, "tests/utils/test_parallel.py::test_wrap_function[ray-local]": 4.386876819999088, - "tests/utils/test_score.py::test_compose_score": 0.003188072001648834, - "tests/utils/test_score.py::test_scorer": 0.006043704999683541, - "tests/utils/test_score.py::test_squashed_r2": 0.00286291600241384, - "tests/utils/test_score.py::test_squashed_variance": 0.002624727998409071, - "tests/utils/test_status.py::test_and_status": 0.0023914820012578275, - "tests/utils/test_status.py::test_not_status": 0.0024304439994011773, - "tests/utils/test_status.py::test_or_status": 0.0031306429991673212, + "tests/utils/test_score.py::test_compose_score": 0.002674737013876438, + "tests/utils/test_score.py::test_scorer": 0.0035687130002770573, + "tests/utils/test_score.py::test_squashed_r2": 0.0024596559815108776, + "tests/utils/test_score.py::test_squashed_variance": 0.0037596760084852576, + "tests/utils/test_status.py::test_and_status": 0.0021642329811584204, + "tests/utils/test_status.py::test_not_status": 0.0017621670267544687, + "tests/utils/test_status.py::test_or_status": 0.0020827190310228616, "tests/utils/test_utility.py::test_cache[2-0-8]": 0.00677607100806199, - "tests/utils/test_utility.py::test_data_utility_learning_wrapper[10-2-0-8]": 0.007143015998735791, - "tests/utils/test_utility.py::test_data_utility_learning_wrapper[2-2-0-8]": 0.008842511999318958, + "tests/utils/test_utility.py::test_data_utility_learning_wrapper[10-2-0-8]": 0.007034309994196519, + "tests/utils/test_utility.py::test_data_utility_learning_wrapper[2-2-0-8]": 0.01102408699807711, "tests/utils/test_utility.py::test_different_cache_signature[model_kwargs0-2-0-8]": 0.0038117940130177885, "tests/utils/test_utility.py::test_different_cache_signature[model_kwargs1-2-0-8]": 0.0034867670328821987, - "tests/utils/test_utility.py::test_different_utility_with_same_cache[2-0-8]": 0.00974041799963743, - "tests/utils/test_utility.py::test_utility_serialization[False-2-0-8]": 0.004299543001252459, - "tests/utils/test_utility.py::test_utility_serialization[True-2-0-8]": 0.004922002000967041, - "tests/utils/test_utility.py::test_utility_show_warnings[4-4-False]": 0.018307410000488744, - "tests/utils/test_utility.py::test_utility_show_warnings[4-4-True]": 0.009923514002366574, - "tests/utils/test_utility.py::test_utility_with_cache[2-0-8]": 0.010860190002858872, - "tests/value/least_core/test_common.py::test_lc_solve_problems[test_game0]": 6.664896995000163, + "tests/utils/test_utility.py::test_different_utility_with_same_cache[2-0-8]": 0.00709219096461311, + "tests/utils/test_utility.py::test_utility_serialization[False-2-0-8]": 0.004958142031682655, + "tests/utils/test_utility.py::test_utility_serialization[True-2-0-8]": 0.004273276019375771, + "tests/utils/test_utility.py::test_utility_show_warnings[4-4-False]": 0.011898054013727233, + "tests/utils/test_utility.py::test_utility_show_warnings[4-4-True]": 0.011095394002040848, + "tests/utils/test_utility.py::test_utility_with_cache[2-0-8]": 0.010787420003907755, + "tests/value/least_core/test_common.py::test_lc_solve_problems[test_game0]": 0.18890138398273848, "tests/value/least_core/test_common.py::test_lc_solve_problems[test_utility0]": 3.0655845460132696, - "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False--1-test_game0-0.1-128]": 0.10254659299971536, - "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False--1-test_game1-0.2-10000]": 0.95324419499957, + "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False--1-test_game0-0.1-128]": 0.05052897802670486, + "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False--1-test_game1-0.2-10000]": 1.0599143210274633, "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False--1-test_utility0-0.1-128]": 0.05090764199849218, "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False--1-test_utility1-0.2-10000]": 0.39550038598827086, - "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False-1-test_game0-0.1-128]": 0.11610117799864383, - "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False-1-test_game1-0.2-10000]": 1.9240173660018627, + "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False-1-test_game0-0.1-128]": 0.045558749028714374, + "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False-1-test_game1-0.2-10000]": 1.016920865047723, "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False-1-test_utility0-0.1-128]": 0.054777625045971945, "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[False-1-test_utility1-0.2-10000]": 0.7125970929628238, - "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True--1-test_game0-0.1-128]": 12.35835815199971, - "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True--1-test_game1-0.2-10000]": 1.27118392400007, + "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True--1-test_game0-0.1-128]": 0.048427555011585355, + "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True--1-test_game1-0.2-10000]": 1.0079815899953246, "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True--1-test_utility0-0.1-128]": 6.515727574034827, "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True--1-test_utility1-0.2-10000]": 0.6112625639943872, - "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True-1-test_game0-0.1-128]": 0.13497778700002527, - "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True-1-test_game1-0.2-10000]": 2.054010283000025, + "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True-1-test_game0-0.1-128]": 0.04923934198450297, + "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True-1-test_game1-0.2-10000]": 1.0299546059977729, "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True-1-test_utility0-0.1-128]": 0.07473104700329714, "tests/value/least_core/test_montecarlo.py::test_montecarlo_least_core[True-1-test_utility1-0.2-10000]": 0.7888634809933137, - "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_game0]": 0.056533884000600665, - "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_game1]": 0.05103961900022114, - "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_game2]": 0.04528383999968355, - "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_game3]": 0.04622581199873821, - "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_game4]": 0.04450138000174775, + "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_game0]": 0.022803050029324368, + "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_game1]": 0.024595678987680003, + "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_game2]": 0.022240159974899143, + "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_game3]": 0.021910707029746845, + "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_game4]": 0.025111915019806474, "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_utility0]": 0.024124946998199448, "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_utility1]": 0.02425819096970372, "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_utility2]": 0.023533977015176788, "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_utility3]": 0.023558928980492055, "tests/value/least_core/test_naive.py::test_naive_least_core[False-test_utility4]": 0.024587185034761205, - "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_game0]": 0.055623405996811925, - "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_game1]": 0.05625994600086415, - "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_game2]": 0.05063546000201313, - "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_game3]": 0.05257723800241365, - "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_game4]": 0.055973189997530426, + "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_game0]": 0.024778541992418468, + "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_game1]": 0.02579708001576364, + "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_game2]": 0.025394369004061446, + "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_game3]": 0.023870316974353045, + "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_game4]": 0.02718632298638113, "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_utility0]": 0.025446541025303304, "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_utility1]": 0.026494102989090607, "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_utility2]": 0.02477889700094238, "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_utility3]": 0.026450325007317588, "tests/value/least_core/test_naive.py::test_naive_least_core[True-test_utility4]": 0.026973432017257437, - "tests/value/loo/test_loo.py::test_loo[100]": 6.34605625200129, - "tests/value/loo/test_loo.py::test_loo[10]": 6.683512068999335, - "tests/value/shapley/test_classwise.py::test_classwise_scorer_accuracies_left_right_margins[101-0.3-0.4]": 0.014495325998723274, - "tests/value/shapley/test_classwise.py::test_classwise_scorer_accuracies_manual_derivation": 0.059531668999625253, - "tests/value/shapley/test_classwise.py::test_classwise_scorer_is_symmetric[101-0.3-0.4]": 0.017718389000947354, - "tests/value/shapley/test_classwise.py::test_classwise_scorer_representation": 0.00893844900019758, - "tests/value/shapley/test_classwise.py::test_classwise_scorer_utility[101-0.3-0.4]": 0.02120917100182851, - "tests/value/shapley/test_classwise.py::test_classwise_shapley[classwise_shapley_exact_solution-n_resample_complement_sets=1-n_samples=500]": 11.03723036699921, - "tests/value/shapley/test_classwise.py::test_classwise_shapley[classwise_shapley_exact_solution_no_default-n_resample_complement_sets=1-n_samples=500]": 12.916025546999663, - "tests/value/shapley/test_classwise.py::test_classwise_shapley[classwise_shapley_exact_solution_no_default_allow_empty_set-n_resample_complement_sets=1-n_samples=500]": 12.068119810999633, - "tests/value/shapley/test_classwise.py::test_classwise_shapley[classwise_shapley_exact_solution_normalized-n_resample_complement_sets=1-n_samples=500]": 10.891289137000058, - "tests/value/shapley/test_classwise.py::test_closed_form_linear_classifier": 0.01344082000105118, - "tests/value/shapley/test_knn.py::test_knn_montecarlo_match": 11.906123751998166, + "tests/value/loo/test_loo.py::test_loo[100]": 2.931964471034007, + "tests/value/loo/test_loo.py::test_loo[10]": 2.289035618014168, + "tests/value/shapley/test_classwise.py::test_classwise_scorer_accuracies_left_right_margins[101-0.3-0.4]": 0.005037697002990171, + "tests/value/shapley/test_classwise.py::test_classwise_scorer_accuracies_manual_derivation": 0.037136607978027314, + "tests/value/shapley/test_classwise.py::test_classwise_scorer_is_symmetric[101-0.3-0.4]": 0.006254975975025445, + "tests/value/shapley/test_classwise.py::test_classwise_scorer_representation": 0.0036907130270265043, + "tests/value/shapley/test_classwise.py::test_classwise_scorer_utility[101-0.3-0.4]": 0.009000057965749875, + "tests/value/shapley/test_classwise.py::test_classwise_shapley[classwise_shapley_exact_solution-n_resample_complement_sets=1-n_samples=500]": 5.354776905995095, + "tests/value/shapley/test_classwise.py::test_classwise_shapley[classwise_shapley_exact_solution_no_default-n_resample_complement_sets=1-n_samples=500]": 6.330135180032812, + "tests/value/shapley/test_classwise.py::test_classwise_shapley[classwise_shapley_exact_solution_no_default_allow_empty_set-n_resample_complement_sets=1-n_samples=500]": 6.155792267003562, + "tests/value/shapley/test_classwise.py::test_classwise_shapley[classwise_shapley_exact_solution_normalized-n_resample_complement_sets=1-n_samples=500]": 5.430430012987927, + "tests/value/shapley/test_classwise.py::test_closed_form_linear_classifier": 0.005138550972333178, + "tests/value/shapley/test_knn.py::test_knn_montecarlo_match": 2.8276229689945467, "tests/value/shapley/test_montecarlo.py::test_analytic_montecarlo_shapley[12-owen-0.1-0.0001-kwargs2]": 0.6999966300209053, "tests/value/shapley/test_montecarlo.py::test_analytic_montecarlo_shapley[12-owen_antithetic-0.1-0.0001-kwargs3]": 1.3923712590476498, "tests/value/shapley/test_montecarlo.py::test_analytic_montecarlo_shapley[12-permutation_montecarlo-0.1-1e-05-kwargs0]": 4.533932764985366, "tests/value/shapley/test_montecarlo.py::test_analytic_montecarlo_shapley[3-group_testing-0.1-0.01-kwargs4]": 2.874565462989267, "tests/value/shapley/test_montecarlo.py::test_analytic_montecarlo_shapley[8-combinatorial_montecarlo-0.2-0.0001-kwargs1]": 4.175152084033471, - "tests/value/shapley/test_montecarlo.py::test_games[combinatorial_montecarlo-0.2-0.0001-kwargs1-test_game0]": 8.304236846999629, - "tests/value/shapley/test_montecarlo.py::test_games[combinatorial_montecarlo-0.2-0.0001-kwargs1-test_game1]": 8.651754697999422, - "tests/value/shapley/test_montecarlo.py::test_games[group_testing-0.1-0.01-kwargs4-test_game0]": 4.506434214001274, - "tests/value/shapley/test_montecarlo.py::test_games[group_testing-0.1-0.01-kwargs4-test_game1]": 5.184473866002008, - "tests/value/shapley/test_montecarlo.py::test_games[owen-0.2-0.0001-kwargs2-test_game0]": 0.695304662000126, - "tests/value/shapley/test_montecarlo.py::test_games[owen-0.2-0.0001-kwargs2-test_game1]": 0.754036617001475, - "tests/value/shapley/test_montecarlo.py::test_games[owen_antithetic-0.1-0.0001-kwargs3-test_game0]": 1.3446016939979017, - "tests/value/shapley/test_montecarlo.py::test_games[owen_antithetic-0.1-0.0001-kwargs3-test_game1]": 1.7906026460022986, - "tests/value/shapley/test_montecarlo.py::test_games[permutation_montecarlo-0.2-0.0001-kwargs0-test_game0]": 9.640759977000926, - "tests/value/shapley/test_montecarlo.py::test_games[permutation_montecarlo-0.2-0.0001-kwargs0-test_game1]": 9.149135870000464, - "tests/value/shapley/test_montecarlo.py::test_grouped_linear_montecarlo_shapley[permutation_montecarlo-kwargs0-scorer0-0.1-2-0-21-2]": 11.869230333000814, - "tests/value/shapley/test_montecarlo.py::test_hoeffding_bound_montecarlo[combinatorial_montecarlo-6-0.1-0.1]": 12.159375920000457, - "tests/value/shapley/test_montecarlo.py::test_hoeffding_bound_montecarlo[permutation_montecarlo-6-0.1-0.1]": 121.21386299999904, + "tests/value/shapley/test_montecarlo.py::test_games[combinatorial_montecarlo-0.2-0.0001-kwargs1-test_game0]": 1.9197496420238167, + "tests/value/shapley/test_montecarlo.py::test_games[combinatorial_montecarlo-0.2-0.0001-kwargs1-test_game1]": 2.5275338669889607, + "tests/value/shapley/test_montecarlo.py::test_games[group_testing-0.1-0.01-kwargs4-test_game0]": 5.158969943964621, + "tests/value/shapley/test_montecarlo.py::test_games[group_testing-0.1-0.01-kwargs4-test_game1]": 6.070634126022924, + "tests/value/shapley/test_montecarlo.py::test_games[owen-0.2-0.0001-kwargs2-test_game0]": 0.5380362030118704, + "tests/value/shapley/test_montecarlo.py::test_games[owen-0.2-0.0001-kwargs2-test_game1]": 0.8358890759991482, + "tests/value/shapley/test_montecarlo.py::test_games[owen_antithetic-0.1-0.0001-kwargs3-test_game0]": 1.2963983940135222, + "tests/value/shapley/test_montecarlo.py::test_games[owen_antithetic-0.1-0.0001-kwargs3-test_game1]": 2.021286355011398, + "tests/value/shapley/test_montecarlo.py::test_games[permutation_montecarlo-0.2-0.0001-kwargs0-test_game0]": 3.723641743970802, + "tests/value/shapley/test_montecarlo.py::test_games[permutation_montecarlo-0.2-0.0001-kwargs0-test_game1]": 3.8597429619985633, + "tests/value/shapley/test_montecarlo.py::test_grouped_linear_montecarlo_shapley[permutation_montecarlo-kwargs0-scorer0-0.1-2-0-21-2]": 5.106256189988926, + "tests/value/shapley/test_montecarlo.py::test_hoeffding_bound_montecarlo[combinatorial_montecarlo-6-0.1-0.1]": 0.001203357009217143, + "tests/value/shapley/test_montecarlo.py::test_hoeffding_bound_montecarlo[permutation_montecarlo-6-0.1-0.1]": 54.87148774100933, "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_shapley[combinatorial_montecarlo-kwargs1-scorer0-0.25-2-0-21]": 17.78464582102606, "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_shapley[group_testing-kwargs4-scorer0-0.25-2-0-21]": 29.239474696019897, "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_shapley[owen-kwargs2-scorer0-0.25-2-0-21]": 4.124498174991459, "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_shapley[owen_antithetic-kwargs3-scorer0-0.25-2-0-21]": 7.887545032019261, "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_shapley[permutation_montecarlo-kwargs0-scorer0-0.25-2-0-21]": 5.8485472809989005, - "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[group_testing-kwargs3-scorer0-0.2-2-0-21]": 105.57146695700249, - "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[owen-kwargs1-scorer0-0.2-2-0-21]": 46.293949323999186, - "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[owen_antithetic-kwargs2-scorer0-0.2-2-0-21]": 75.77437868900051, - "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[permutation_montecarlo-kwargs0-scorer0-0.2-2-0-21]": 14.84272324000085, + "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[group_testing-kwargs3-scorer0-0.2-2-0-21]": 99.55651604299783, + "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[owen-kwargs1-scorer0-0.2-2-0-21]": 34.54854124702979, + "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[owen_antithetic-kwargs2-scorer0-0.2-2-0-21]": 68.9907481589762, + "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[permutation_montecarlo-kwargs0-scorer0-0.2-2-0-21]": 6.065520199976163, "tests/value/shapley/test_montecarlo.py::test_montecarlo_shapley_housing_dataset[12-3-12-combinatorial_montecarlo-kwargs0]": 0.16786966001382098, "tests/value/shapley/test_montecarlo.py::test_montecarlo_shapley_housing_dataset[12-3-12-owen-kwargs1]": 17.011920137971174, "tests/value/shapley/test_montecarlo.py::test_montecarlo_shapley_housing_dataset[12-3-12-owen_antithetic-kwargs2]": 35.88025256394758, "tests/value/shapley/test_montecarlo.py::test_montecarlo_shapley_housing_dataset[12-3-4-group_testing-kwargs3]": 0.25901710899779573, - "tests/value/shapley/test_montecarlo.py::test_seed[combinatorial_montecarlo-kwargs0-test_game0]": 0.10237690700159874, - "tests/value/shapley/test_montecarlo.py::test_seed[group_testing-kwargs3-test_game0]": 1.3946212869996089, - "tests/value/shapley/test_montecarlo.py::test_seed[owen-kwargs1-test_game0]": 2.984055114999137, - "tests/value/shapley/test_montecarlo.py::test_seed[owen_antithetic-kwargs2-test_game0]": 5.702334433002761, + "tests/value/shapley/test_montecarlo.py::test_seed[combinatorial_montecarlo-kwargs0-test_game0]": 0.015400617005070671, + "tests/value/shapley/test_montecarlo.py::test_seed[group_testing-kwargs3-test_game0]": 0.4990526599576697, + "tests/value/shapley/test_montecarlo.py::test_seed[owen-kwargs1-test_game0]": 2.7092463899753056, + "tests/value/shapley/test_montecarlo.py::test_seed[owen_antithetic-kwargs2-test_game0]": 6.592550466011744, "tests/value/shapley/test_naive.py::test_analytic_exact_shapley[12-combinatorial_exact_shapley-0.01-1e-05]": 2.798590613005217, "tests/value/shapley/test_naive.py::test_analytic_exact_shapley[6-permutation_exact_shapley-0.01-1e-05]": 0.34537768000154756, - "tests/value/shapley/test_naive.py::test_games[combinatorial_exact_shapley-test_game0-0.1-1e-05]": 0.035801175001324737, - "tests/value/shapley/test_naive.py::test_games[combinatorial_exact_shapley-test_game1-0.1-1e-05]": 0.020296718997997232, - "tests/value/shapley/test_naive.py::test_games[combinatorial_exact_shapley-test_game2-0.1-1e-05]": 0.026713223998740432, - "tests/value/shapley/test_naive.py::test_games[combinatorial_exact_shapley-test_game3-0.1-1e-05]": 0.024250888998722075, - "tests/value/shapley/test_naive.py::test_games[combinatorial_exact_shapley-test_game4-0.1-1e-05]": 0.08578255800057377, - "tests/value/shapley/test_naive.py::test_games[permutation_exact_shapley-test_game0-0.1-1e-05]": 0.0334680340019986, - "tests/value/shapley/test_naive.py::test_games[permutation_exact_shapley-test_game1-0.1-1e-05]": 0.02248540199798299, - "tests/value/shapley/test_naive.py::test_games[permutation_exact_shapley-test_game2-0.1-1e-05]": 0.021266358000502805, - "tests/value/shapley/test_naive.py::test_games[permutation_exact_shapley-test_game3-0.1-1e-05]": 0.02347195299989835, - "tests/value/shapley/test_naive.py::test_games[permutation_exact_shapley-test_game4-0.1-1e-05]": 0.9876527700016595, - "tests/value/shapley/test_naive.py::test_grouped_linear[2-0-50-3-r2]": 0.19884431500031496, - "tests/value/shapley/test_naive.py::test_grouped_linear[2-1-100-5-explained_variance]": 3.9915946569999505, - "tests/value/shapley/test_naive.py::test_grouped_linear[2-1-100-5-r2]": 3.9885682109998015, + "tests/value/shapley/test_naive.py::test_games[combinatorial_exact_shapley-test_game0-0.1-1e-05]": 0.013095707981847227, + "tests/value/shapley/test_naive.py::test_games[combinatorial_exact_shapley-test_game1-0.1-1e-05]": 0.010749600012786686, + "tests/value/shapley/test_naive.py::test_games[combinatorial_exact_shapley-test_game2-0.1-1e-05]": 0.009951793035725132, + "tests/value/shapley/test_naive.py::test_games[combinatorial_exact_shapley-test_game3-0.1-1e-05]": 0.0090146989969071, + "tests/value/shapley/test_naive.py::test_games[combinatorial_exact_shapley-test_game4-0.1-1e-05]": 0.03628412194666453, + "tests/value/shapley/test_naive.py::test_games[permutation_exact_shapley-test_game0-0.1-1e-05]": 0.017009111965307966, + "tests/value/shapley/test_naive.py::test_games[permutation_exact_shapley-test_game1-0.1-1e-05]": 0.010189941996941343, + "tests/value/shapley/test_naive.py::test_games[permutation_exact_shapley-test_game2-0.1-1e-05]": 0.01022218499565497, + "tests/value/shapley/test_naive.py::test_games[permutation_exact_shapley-test_game3-0.1-1e-05]": 0.009432713995920494, + "tests/value/shapley/test_naive.py::test_games[permutation_exact_shapley-test_game4-0.1-1e-05]": 0.5037095320003573, + "tests/value/shapley/test_naive.py::test_grouped_linear[2-0-50-3-r2]": 0.11044956598198041, + "tests/value/shapley/test_naive.py::test_grouped_linear[2-1-100-5-explained_variance]": 2.3892422189819627, + "tests/value/shapley/test_naive.py::test_grouped_linear[2-1-100-5-r2]": 2.3473940060066525, "tests/value/shapley/test_naive.py::test_linear[2-0-10-r2]": 0.05533879197901115, "tests/value/shapley/test_naive.py::test_linear[2-1-10-explained_variance]": 0.058987755968701094, "tests/value/shapley/test_naive.py::test_linear[2-1-10-neg_median_absolute_error]": 0.05515471697435714, "tests/value/shapley/test_naive.py::test_linear[2-1-10-r2]": 0.05683578198659234, - "tests/value/shapley/test_naive.py::test_linear_with_outlier[2-0-20-r2]": 25.743576199000017, - "tests/value/shapley/test_naive.py::test_linear_with_outlier[2-1-20-explained_variance]": 26.06965675200081, - "tests/value/shapley/test_naive.py::test_linear_with_outlier[2-1-20-neg_median_absolute_error]": 25.645237798999005, - "tests/value/shapley/test_naive.py::test_linear_with_outlier[2-1-20-r2]": 25.97635805399841, - "tests/value/shapley/test_naive.py::test_polynomial[coefficients0-r2]": 0.20116403300016827, - "tests/value/shapley/test_naive.py::test_polynomial[coefficients1-neg_median_absolute_error]": 0.20279847600068024, - "tests/value/shapley/test_naive.py::test_polynomial[coefficients2-explained_variance]": 0.20646126699830347, - "tests/value/shapley/test_naive.py::test_polynomial_with_outlier[coefficients0-r2]": 0.15503699600049003, - "tests/value/shapley/test_naive.py::test_polynomial_with_outlier[coefficients1-neg_median_absolute_error]": 0.15186486699894886, - "tests/value/shapley/test_naive.py::test_polynomial_with_outlier[coefficients2-explained_variance]": 0.1560443580001447, - "tests/value/shapley/test_truncated.py::test_games[done0-NoTruncation-truncation_kwargs0-test_game0]": 8.864981821001493, - "tests/value/shapley/test_truncated.py::test_games[done0-NoTruncation-truncation_kwargs0-test_game1]": 8.904717276998781, - "tests/value/shapley/test_truncated.py::test_games[done1-FixedTruncation-truncation_kwargs1-test_game0]": 8.893666212001335, - "tests/value/shapley/test_truncated.py::test_games[done1-FixedTruncation-truncation_kwargs1-test_game1]": 8.871429693997925, + "tests/value/shapley/test_naive.py::test_linear_with_outlier[2-0-20-r2]": 15.295730330020888, + "tests/value/shapley/test_naive.py::test_linear_with_outlier[2-1-20-explained_variance]": 15.311946107976837, + "tests/value/shapley/test_naive.py::test_linear_with_outlier[2-1-20-neg_median_absolute_error]": 15.246478994027711, + "tests/value/shapley/test_naive.py::test_linear_with_outlier[2-1-20-r2]": 15.122833114990499, + "tests/value/shapley/test_naive.py::test_polynomial[coefficients0-r2]": 0.12686681901686825, + "tests/value/shapley/test_naive.py::test_polynomial[coefficients1-neg_median_absolute_error]": 0.11067792598623782, + "tests/value/shapley/test_naive.py::test_polynomial[coefficients2-explained_variance]": 0.11747662501875311, + "tests/value/shapley/test_naive.py::test_polynomial_with_outlier[coefficients0-r2]": 0.08565683299093507, + "tests/value/shapley/test_naive.py::test_polynomial_with_outlier[coefficients1-neg_median_absolute_error]": 0.08773901200038381, + "tests/value/shapley/test_naive.py::test_polynomial_with_outlier[coefficients2-explained_variance]": 0.09032825799658895, + "tests/value/shapley/test_truncated.py::test_games[done0-NoTruncation-truncation_kwargs0-test_game0]": 3.9726052110199817, + "tests/value/shapley/test_truncated.py::test_games[done0-NoTruncation-truncation_kwargs0-test_game1]": 4.48901929197018, + "tests/value/shapley/test_truncated.py::test_games[done1-FixedTruncation-truncation_kwargs1-test_game0]": 4.009335360024124, + "tests/value/shapley/test_truncated.py::test_games[done1-FixedTruncation-truncation_kwargs1-test_game1]": 4.109410956996726, "tests/value/shapley/test_truncated.py::test_tmcs_analytic_montecarlo_shapley[12-truncated_montecarlo-0.1-1e-05-kwargs0]": 5.025441929989029, "tests/value/shapley/test_truncated.py::test_tmcs_linear_montecarlo_shapley[truncated_montecarlo-kwargs0-scorer0-0.25-2-0-21]": 5.633914494974306, - "tests/value/shapley/test_truncated.py::test_tmcs_linear_montecarlo_with_outlier[truncated_montecarlo-kwargs0-scorer0-0.2-2-0-21]": 7.1438663650005765, - "tests/value/test_sampler.py::test_chunkify[AntitheticSampler]": 0.002635386001202278, - "tests/value/test_sampler.py::test_chunkify[DeterministicUniformSampler]": 0.002136322002115776, - "tests/value/test_sampler.py::test_chunkify[RandomHierarchicalSampler]": 0.0024412720013060607, - "tests/value/test_sampler.py::test_chunkify[UniformSampler]": 0.0022573409987671766, - "tests/value/test_sampler.py::test_chunkify_permutation[DeterministicPermutationSampler]": 0.0024367070000153035, - "tests/value/test_sampler.py::test_chunkify_permutation[PermutationSampler]": 0.002322892001757282, - "tests/value/test_sampler.py::test_proper[indices0-AntitheticSampler]": 0.003334062997964793, - "tests/value/test_sampler.py::test_proper[indices0-DeterministicPermutationSampler]": 0.002626270001201192, - "tests/value/test_sampler.py::test_proper[indices0-DeterministicUniformSampler]": 0.002812078997521894, - "tests/value/test_sampler.py::test_proper[indices0-PermutationSampler]": 0.002539194001656142, - "tests/value/test_sampler.py::test_proper[indices0-RandomHierarchicalSampler]": 0.0026362519984104438, - "tests/value/test_sampler.py::test_proper[indices0-UniformSampler]": 0.0024412409984506667, - "tests/value/test_sampler.py::test_proper[indices1-AntitheticSampler]": 0.0027277339995634975, - "tests/value/test_sampler.py::test_proper[indices1-DeterministicPermutationSampler]": 0.002861182998458389, - "tests/value/test_sampler.py::test_proper[indices1-DeterministicUniformSampler]": 0.004058188998897094, - "tests/value/test_sampler.py::test_proper[indices1-PermutationSampler]": 0.0026329000011173775, - "tests/value/test_sampler.py::test_proper[indices1-RandomHierarchicalSampler]": 0.003700332001244533, - "tests/value/test_sampler.py::test_proper[indices1-UniformSampler]": 0.003620775998570025, - "tests/value/test_sampler.py::test_proper_reproducible[indices0-AntitheticSampler]": 0.0028454019975470146, - "tests/value/test_sampler.py::test_proper_reproducible[indices0-PermutationSampler]": 0.0024918920007621637, - "tests/value/test_sampler.py::test_proper_reproducible[indices0-RandomHierarchicalSampler]": 0.0020272490019124234, - "tests/value/test_sampler.py::test_proper_reproducible[indices0-UniformSampler]": 0.0027337100000295322, - "tests/value/test_sampler.py::test_proper_reproducible[indices1-AntitheticSampler]": 0.009103345002586138, - "tests/value/test_sampler.py::test_proper_reproducible[indices1-PermutationSampler]": 0.003312619001007988, - "tests/value/test_sampler.py::test_proper_reproducible[indices1-RandomHierarchicalSampler]": 0.017666732001089258, - "tests/value/test_sampler.py::test_proper_reproducible[indices1-UniformSampler]": 0.010906160998274572, - "tests/value/test_sampler.py::test_proper_stochastic[indices0-AntitheticSampler]": 0.0025616729999455856, - "tests/value/test_sampler.py::test_proper_stochastic[indices0-PermutationSampler]": 0.0034559460000309628, - "tests/value/test_sampler.py::test_proper_stochastic[indices0-RandomHierarchicalSampler]": 0.0029194710004958324, - "tests/value/test_sampler.py::test_proper_stochastic[indices0-UniformSampler]": 0.0028906579991598846, - "tests/value/test_sampler.py::test_proper_stochastic[indices1-AntitheticSampler]": 0.011208809999516234, - "tests/value/test_sampler.py::test_proper_stochastic[indices1-PermutationSampler]": 0.003227124001568882, - "tests/value/test_sampler.py::test_proper_stochastic[indices1-RandomHierarchicalSampler]": 0.020847252999374177, - "tests/value/test_sampler.py::test_proper_stochastic[indices1-UniformSampler]": 0.01049548499941011, - "tests/value/test_semivalues.py::test_banzhaf[AntitheticPermutationSampler-5]": 19.099751196999932, - "tests/value/test_semivalues.py::test_banzhaf[AntitheticSampler-5]": 8.640272729999197, - "tests/value/test_semivalues.py::test_banzhaf[DeterministicPermutationSampler-5]": 11.046467014999507, - "tests/value/test_semivalues.py::test_banzhaf[DeterministicUniformSampler-5]": 7.140763282997796, - "tests/value/test_semivalues.py::test_banzhaf[PermutationSampler-5]": 16.536335553000754, - "tests/value/test_semivalues.py::test_banzhaf[UniformSampler-5]": 8.56469571100206, - "tests/value/test_semivalues.py::test_coefficients[banzhaf_coefficient-100]": 0.01005963700117718, - "tests/value/test_semivalues.py::test_coefficients[banzhaf_coefficient-10]": 0.008440342002359102, - "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w0-100]": 0.01072616300007212, - "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w0-10]": 0.010928496998531045, - "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w1-100]": 0.012177771001006477, - "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w1-10]": 0.00821317400004773, - "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w2-100]": 0.010945971000182908, - "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w2-10]": 0.008208530998672359, - "tests/value/test_semivalues.py::test_coefficients[shapley_coefficient-100]": 0.019708362000528723, - "tests/value/test_semivalues.py::test_coefficients[shapley_coefficient-10]": 0.007813238997187, - "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-AntitheticPermutationSampler-test_game0]": 22.607437191998542, - "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-AntitheticPermutationSampler-test_game1]": 19.905466008996882, - "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-AntitheticSampler-test_game0]": 22.902231953998125, - "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-AntitheticSampler-test_game1]": 20.254530511001576, - "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-PermutationSampler-test_game0]": 22.228997524001898, - "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-PermutationSampler-test_game1]": 19.948070817999906, - "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-UniformSampler-test_game0]": 23.313307015001556, - "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-UniformSampler-test_game1]": 20.214418551000563, - "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-AntitheticPermutationSampler-test_game0]": 16.339908187999754, - "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-AntitheticPermutationSampler-test_game1]": 14.935287896998489, - "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-AntitheticSampler-test_game0]": 16.71660759900078, - "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-AntitheticSampler-test_game1]": 15.118247157999576, - "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-PermutationSampler-test_game0]": 16.669900056000188, - "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-PermutationSampler-test_game1]": 14.85890512199876, - "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-UniformSampler-test_game0]": 16.9996823649999, - "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-UniformSampler-test_game1]": 15.419395829998393, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicPermutationSampler-test_game0]": 7.571815403000073, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicPermutationSampler-test_game1]": 6.795873736999056, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicPermutationSampler-test_game2]": 6.49785933900057, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicPermutationSampler-test_game3]": 7.046587265998824, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicUniformSampler-test_game0]": 6.9995765299991035, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicUniformSampler-test_game1]": 7.470778629000051, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicUniformSampler-test_game2]": 6.813381661997482, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicUniformSampler-test_game3]": 7.335269874001824, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicPermutationSampler-test_game0]": 8.675189851999676, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicPermutationSampler-test_game1]": 6.932035337997149, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicPermutationSampler-test_game2]": 6.9341853499990975, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicPermutationSampler-test_game3]": 6.737996050998845, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicUniformSampler-test_game0]": 4.491834778002158, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicUniformSampler-test_game1]": 6.446436399000959, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicUniformSampler-test_game2]": 6.968900550000399, - "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicUniformSampler-test_game3]": 6.659720210998785, - "tests/value/test_semivalues.py::test_marginal_batch_size[PermutationSampler-beta_coefficient_w-5-test_game0]": 0.004239154999595485, + "tests/value/shapley/test_truncated.py::test_tmcs_linear_montecarlo_with_outlier[truncated_montecarlo-kwargs0-scorer0-0.2-2-0-21]": 3.0232558990246616, + "tests/value/test_sampler.py::test_chunkify[AntitheticSampler]": 0.002352299023186788, + "tests/value/test_sampler.py::test_chunkify[DeterministicUniformSampler]": 0.002171223022742197, + "tests/value/test_sampler.py::test_chunkify[RandomHierarchicalSampler]": 0.0020702430338133126, + "tests/value/test_sampler.py::test_chunkify[UniformSampler]": 0.0028199770022183657, + "tests/value/test_sampler.py::test_chunkify_permutation[DeterministicPermutationSampler]": 0.002369680005358532, + "tests/value/test_sampler.py::test_chunkify_permutation[PermutationSampler]": 0.002037067955825478, + "tests/value/test_sampler.py::test_proper[indices0-AntitheticSampler]": 0.002861024026060477, + "tests/value/test_sampler.py::test_proper[indices0-DeterministicPermutationSampler]": 0.002655454009072855, + "tests/value/test_sampler.py::test_proper[indices0-DeterministicUniformSampler]": 0.003174368990585208, + "tests/value/test_sampler.py::test_proper[indices0-PermutationSampler]": 0.0024646950187161565, + "tests/value/test_sampler.py::test_proper[indices0-RandomHierarchicalSampler]": 0.0023855049803387374, + "tests/value/test_sampler.py::test_proper[indices0-UniformSampler]": 0.0028759060078300536, + "tests/value/test_sampler.py::test_proper[indices1-AntitheticSampler]": 0.003161819011438638, + "tests/value/test_sampler.py::test_proper[indices1-DeterministicPermutationSampler]": 0.002913616015575826, + "tests/value/test_sampler.py::test_proper[indices1-DeterministicUniformSampler]": 0.00358530500670895, + "tests/value/test_sampler.py::test_proper[indices1-PermutationSampler]": 0.0029439219797495753, + "tests/value/test_sampler.py::test_proper[indices1-RandomHierarchicalSampler]": 0.004866360017331317, + "tests/value/test_sampler.py::test_proper[indices1-UniformSampler]": 0.003348411002662033, + "tests/value/test_sampler.py::test_proper_reproducible[indices0-AntitheticSampler]": 0.0021045640169177204, + "tests/value/test_sampler.py::test_proper_reproducible[indices0-PermutationSampler]": 0.002255333965877071, + "tests/value/test_sampler.py::test_proper_reproducible[indices0-RandomHierarchicalSampler]": 0.002645997970830649, + "tests/value/test_sampler.py::test_proper_reproducible[indices0-UniformSampler]": 0.002439948031678796, + "tests/value/test_sampler.py::test_proper_reproducible[indices1-AntitheticSampler]": 0.012846849975176156, + "tests/value/test_sampler.py::test_proper_reproducible[indices1-PermutationSampler]": 0.0033622210030443966, + "tests/value/test_sampler.py::test_proper_reproducible[indices1-RandomHierarchicalSampler]": 0.016803934995550662, + "tests/value/test_sampler.py::test_proper_reproducible[indices1-UniformSampler]": 0.009853368013864383, + "tests/value/test_sampler.py::test_proper_stochastic[indices0-AntitheticSampler]": 0.002318738988833502, + "tests/value/test_sampler.py::test_proper_stochastic[indices0-PermutationSampler]": 0.0024791909963823855, + "tests/value/test_sampler.py::test_proper_stochastic[indices0-RandomHierarchicalSampler]": 0.0026730689860414714, + "tests/value/test_sampler.py::test_proper_stochastic[indices0-UniformSampler]": 0.0024485259782522917, + "tests/value/test_sampler.py::test_proper_stochastic[indices1-AntitheticSampler]": 0.011456050968263298, + "tests/value/test_sampler.py::test_proper_stochastic[indices1-PermutationSampler]": 0.004046668007504195, + "tests/value/test_sampler.py::test_proper_stochastic[indices1-RandomHierarchicalSampler]": 0.017887227004393935, + "tests/value/test_sampler.py::test_proper_stochastic[indices1-UniformSampler]": 0.013727245997870341, + "tests/value/test_semivalues.py::test_banzhaf[AntitheticPermutationSampler-5]": 8.8424951180059, + "tests/value/test_semivalues.py::test_banzhaf[AntitheticSampler-5]": 3.934585579962004, + "tests/value/test_semivalues.py::test_banzhaf[DeterministicPermutationSampler-5]": 4.930636252975091, + "tests/value/test_semivalues.py::test_banzhaf[DeterministicUniformSampler-5]": 3.3139901740069035, + "tests/value/test_semivalues.py::test_banzhaf[PermutationSampler-5]": 7.1331721209862735, + "tests/value/test_semivalues.py::test_banzhaf[UniformSampler-5]": 3.9336799690208863, + "tests/value/test_semivalues.py::test_coefficients[banzhaf_coefficient-100]": 0.004235609987517819, + "tests/value/test_semivalues.py::test_coefficients[banzhaf_coefficient-10]": 0.0035420969943515956, + "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w0-100]": 0.004731877997983247, + "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w0-10]": 0.004268383985618129, + "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w1-100]": 0.00397408299613744, + "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w1-10]": 0.0036294079618528485, + "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w2-100]": 0.004120993980905041, + "tests/value/test_semivalues.py::test_coefficients[beta_coefficient_w2-10]": 0.003885133017320186, + "tests/value/test_semivalues.py::test_coefficients[shapley_coefficient-100]": 0.004692697984864935, + "tests/value/test_semivalues.py::test_coefficients[shapley_coefficient-10]": 0.0032941289828158915, + "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-AntitheticPermutationSampler-test_game0]": 10.938828482991084, + "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-AntitheticPermutationSampler-test_game1]": 9.457341374014504, + "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-AntitheticSampler-test_game0]": 11.300962019013241, + "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-AntitheticSampler-test_game1]": 9.835019734979142, + "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-PermutationSampler-test_game0]": 10.733634837990394, + "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-PermutationSampler-test_game1]": 9.321977876999881, + "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-UniformSampler-test_game0]": 11.094729618023848, + "tests/value/test_semivalues.py::test_games_shapley[beta_coefficient_w-UniformSampler-test_game1]": 9.7448798789992, + "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-AntitheticPermutationSampler-test_game0]": 7.972334994003177, + "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-AntitheticPermutationSampler-test_game1]": 6.824364921019878, + "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-AntitheticSampler-test_game0]": 8.056306425016373, + "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-AntitheticSampler-test_game1]": 7.130653911037371, + "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-PermutationSampler-test_game0]": 7.846854715025984, + "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-PermutationSampler-test_game1]": 6.972198699979344, + "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-UniformSampler-test_game0]": 8.307511443999829, + "tests/value/test_semivalues.py::test_games_shapley[shapley_coefficient-UniformSampler-test_game1]": 8.480023446987616, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicPermutationSampler-test_game0]": 3.3534141020209063, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicPermutationSampler-test_game1]": 2.9377814330218825, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicPermutationSampler-test_game2]": 3.0197440950141754, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicPermutationSampler-test_game3]": 3.065462252998259, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicUniformSampler-test_game0]": 2.530866265995428, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicUniformSampler-test_game1]": 2.936823854019167, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicUniformSampler-test_game2]": 3.0397010059969034, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[beta_coefficient_w-DeterministicUniformSampler-test_game3]": 2.9198287760373205, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicPermutationSampler-test_game0]": 2.0015553509874735, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicPermutationSampler-test_game1]": 3.1432031750155147, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicPermutationSampler-test_game2]": 2.8129150270251557, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicPermutationSampler-test_game3]": 3.019237435044488, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicUniformSampler-test_game0]": 2.2434276760031935, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicUniformSampler-test_game1]": 3.035445230983896, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicUniformSampler-test_game2]": 2.9089672839909326, + "tests/value/test_semivalues.py::test_games_shapley_deterministic[shapley_coefficient-DeterministicUniformSampler-test_game3]": 3.0945032040181104, + "tests/value/test_semivalues.py::test_marginal_batch_size[PermutationSampler-beta_coefficient_w-5-test_game0]": 0.006125468004029244, "tests/value/test_semivalues.py::test_shapley[beta_coefficient_w-AntitheticPermutationSampler-5]": 5.1298250389809255, "tests/value/test_semivalues.py::test_shapley[beta_coefficient_w-AntitheticSampler-5]": 21.97495059997891, "tests/value/test_semivalues.py::test_shapley[beta_coefficient_w-DeterministicPermutationSampler-5]": 5.294114143965999, @@ -835,20 +969,20 @@ "tests/value/test_semivalues.py::test_shapley[shapley_coefficient-DeterministicUniformSampler-5]": 3.263753114035353, "tests/value/test_semivalues.py::test_shapley[shapley_coefficient-PermutationSampler-5]": 4.766259174008155, "tests/value/test_semivalues.py::test_shapley[shapley_coefficient-UniformSampler-5]": 8.919797526003094, - "tests/value/test_semivalues.py::test_shapley_batch_size[1-PermutationSampler-beta_coefficient_w-5-test_game0]": 9.699354351001602, - "tests/value/test_semivalues.py::test_shapley_batch_size[2-PermutationSampler-beta_coefficient_w-5-test_game0]": 11.229309665000983, + "tests/value/test_semivalues.py::test_shapley_batch_size[1-PermutationSampler-beta_coefficient_w-5-test_game0]": 5.223603894002736, + "tests/value/test_semivalues.py::test_shapley_batch_size[2-PermutationSampler-beta_coefficient_w-5-test_game0]": 5.565272415027721, "tests/value/test_semivalues.py::test_shapley_batch_size[5-PermutationSampler-beta_coefficient_w-5]": 9.19877936199191, - "tests/value/test_stopping.py::test_history_deviation[0.01-100]": 1.7738857549993554, - "tests/value/test_stopping.py::test_history_deviation[0.01-1]": 0.029810868998538353, - "tests/value/test_stopping.py::test_history_deviation[0.01-42]": 0.7947784120024153, - "tests/value/test_stopping.py::test_history_deviation[0.05-100]": 0.3636526160007634, - "tests/value/test_stopping.py::test_history_deviation[0.05-1]": 0.010319109000192839, - "tests/value/test_stopping.py::test_history_deviation[0.05-42]": 0.16107529900000372, - "tests/value/test_stopping.py::test_make_criterion": 0.016543962998184725, - "tests/value/test_stopping.py::test_max_checks": 0.006280684001467307, - "tests/value/test_stopping.py::test_max_time": 0.30847623600129737, - "tests/value/test_stopping.py::test_minmax_updates": 0.012927236997711589, - "tests/value/test_stopping.py::test_standard_error": 0.007960140001159743, - "tests/value/test_stopping.py::test_stopping_criterion": 0.011265246001130436, - "tests/value/test_stopping.py::test_stopping_criterion_composition": 0.019021763000637293 + "tests/value/test_stopping.py::test_history_deviation[0.01-100]": 0.9386799350031652, + "tests/value/test_stopping.py::test_history_deviation[0.01-1]": 0.016687404015101492, + "tests/value/test_stopping.py::test_history_deviation[0.01-42]": 0.34465136696235277, + "tests/value/test_stopping.py::test_history_deviation[0.05-100]": 0.197514128027251, + "tests/value/test_stopping.py::test_history_deviation[0.05-1]": 0.005374073953134939, + "tests/value/test_stopping.py::test_history_deviation[0.05-42]": 0.08412943602888845, + "tests/value/test_stopping.py::test_make_criterion": 0.005493109958479181, + "tests/value/test_stopping.py::test_max_checks": 0.0034617370110936463, + "tests/value/test_stopping.py::test_max_time": 0.30653654897469096, + "tests/value/test_stopping.py::test_minmax_updates": 0.0035910180013161153, + "tests/value/test_stopping.py::test_standard_error": 0.00415661497390829, + "tests/value/test_stopping.py::test_stopping_criterion": 0.0028504370129667222, + "tests/value/test_stopping.py::test_stopping_criterion_composition": 0.005712636018870398 } \ No newline at end of file diff --git a/docs/getting-started/advanced-usage.md b/docs/getting-started/advanced-usage.md new file mode 100644 index 000000000..4a865436c --- /dev/null +++ b/docs/getting-started/advanced-usage.md @@ -0,0 +1,190 @@ +--- +title: Advanced usage +alias: + name: advanced-usage + text: Advanced usage +--- + +# Advanced usage + +Besides the dos and don'ts of data valuation itself, which are the subject of +the examples and the documentation of each method, there are two main things to +keep in mind when using pyDVL namely Parallelization and Caching. + +## Parallelization { #setting-up-parallelization } + +pyDVL uses parallelization to scale and speed up computations. It does so +using one of Dask, Ray or Joblib. The first is used in +the [influence][pydvl.influence] package whereas the other two +are used in the [value][pydvl.value] package. + +### Data valuation + +For data valuation, pyDVL uses [joblib](https://joblib.readthedocs.io/en/latest/) for local +parallelization (within one machine) and supports using +[Ray](https://ray.io) for distributed parallelization (across multiple machines). + +The former works out of the box but for the latter you will need to install +additional dependencies (see [Extras][installation-extras]) +and to provide a running cluster (or run ray in local mode). + +!!! info + + As of v0.9.0 pyDVL does not allow requesting resources per task sent to the + cluster, so you will need to make sure that each worker has enough resources to + handle the tasks it receives. A data valuation task using game-theoretic methods + will typically make a copy of the whole model and dataset to each worker, even + if the re-training only happens on a subset of the data. This means that you + should make sure that each worker has enough memory to handle the whole dataset. + +#### Joblib + +Please follow the instructions in Joblib's documentation +for all possible configuration options that you can pass to the +[parallel_config][joblib.parallel_config] context manager. + +To use the joblib parallel backend with the `loky` backend and verbosity set to `100` +to compute exact shapley values you would use: + +```python +import joblib +from pydvl.parallel import ParallelConfig +from pydvl.value.shapley import combinatorial_exact_shapley +from pydvl.utils.utility import Utility + +config = ParallelConfig(backend="joblib") +u = Utility(...) + +with joblib.parallel_config(backend="loky", verbose=100): + combinatorial_exact_shapley(u, config=config) +``` + +#### Ray + +Please follow the instructions in Ray's documentation to +[set up a remote cluster](https://docs.ray.io/en/latest/cluster/key-concepts.html). +You could alternatively use a local cluster and in that case you don't have to set +anything up. + +Before starting a computation, you should initialize ray by calling +[`ray.init`][ray.init] with the appropriate parameters: + +To set up and start a local ray cluster with 4 CPUs you would use: + +```python +import ray + +ray.init(num_cpus=4) +``` + +Whereas for a remote ray cluster you would use: + +```python +import ray + +address = "" +ray.init(address) +``` + +To use the ray parallel backend to compute exact shapley values you would use: + +```python +import ray +from pydvl.parallel import ParallelConfig +from pydvl.value.shapley import combinatorial_exact_shapley +from pydvl.utils.utility import Utility + +ray.init() +config = ParallelConfig(backend="ray") +u = Utility(...) +combinatorial_exact_shapley(u, config=config) +``` + +### Influence functions + +Refer to the [[scaling-influence-computations]] page for explanations +about parallelization for Influence Function computations. + +## Caching { #getting-started-cache } + +PyDVL can cache (memoize) the computation of the utility function +and speed up some computations for data valuation. +It is however disabled by default. +When it is enabled it takes into account the data indices passed as argument +and the utility function wrapped into the +[Utility][pydvl.utils.utility.Utility] object. This means that +care must be taken when reusing the same utility function with different data, +see the documentation for the [caching package][pydvl.utils.caching] for more +information. + +In general, caching won't play a major role in the computation of Shapley values +because the probability of sampling the same subset twice, and hence needing +the same utility function computation, is very low. However, it can be very +useful when comparing methods that use the same utility function, or when +running multiple experiments with the same data. + +pyDVL supports 3 different caching backends: + +- [InMemoryCacheBackend][pydvl.utils.caching.memory.InMemoryCacheBackend]: + an in-memory cache backend that uses a dictionary to store and retrieve + cached values. This is used to share cached values between threads + in a single process. + +- [DiskCacheBackend][pydvl.utils.caching.disk.DiskCacheBackend]: + a disk-based cache backend that uses pickled values written to and read from disk. + This is used to share cached values between processes in a single machine. +- [MemcachedCacheBackend][pydvl.utils.caching.memcached.MemcachedCacheBackend]: + a [Memcached](https://memcached.org/)-based cache backend that uses pickled values written to + and read from a Memcached server. This is used to share cached values + between processes across multiple machines. + + ??? info "Memcached extras" + + The Memcached backend requires optional dependencies. + See [Extras][installation-extras] for more information. + +As an example, here's how one would use the disk-based cached backend +with a utility: + +```python +from pydvl.utils.caching.disk import DiskCacheBackend +from pydvl.utils.utility import Utility + +cache_backend = DiskCacheBackend() +u = Utility(..., cache_backend=cache_backend) +``` + +Please refer to the documentation and examples of each backend class for more details. + +!!! tip "When is the cache really necessary?" + Crucially, semi-value computations with the + [PermutationSampler][pydvl.value.sampler.PermutationSampler] require caching + to be enabled, or they will take twice as long as the direct implementation + in [compute_shapley_values][pydvl.value.shapley.compute_shapley_values]. + +!!! tip "Using the cache" + Continue reading about the cache in the documentation + for the [caching package][pydvl.utils.caching]. + +### Setting up the Memcached cache { #setting-up-memcached } + +[Memcached](https://memcached.org/) is an in-memory key-value store accessible +over the network. pyDVL can use it to cache the computation of the utility function +and speed up some computations (in particular, semi-value computations with the +[PermutationSampler][pydvl.value.sampler.PermutationSampler] but other methods +may benefit as well). + +You can either install it as a package or run it inside a docker container (the +simplest). For installation instructions, refer to the [Getting +started](https://github.com/memcached/memcached/wiki#getting-started) section in +memcached's wiki. Then you can run it with: + +```shell +memcached -u user +``` + +To run memcached inside a container in daemon mode instead, use: + +```shell +docker container run -d --rm -p 11211:11211 memcached:latest +``` diff --git a/docs/getting-started/applications.md b/docs/getting-started/applications.md index ee3717c6f..87c2e0fcf 100644 --- a/docs/getting-started/applications.md +++ b/docs/getting-started/applications.md @@ -23,7 +23,7 @@ comprehensive overview, along with concrete examples, please refer to the [Transferlab blog post]({{ transferlab.website }}blog/data-valuation-applications/) on this topic. -## Data Engineering +## Data engineering Some of the promising applications in data engineering include: diff --git a/docs/getting-started/first-steps.md b/docs/getting-started/first-steps.md index 8f198e38b..01a8b792b 100644 --- a/docs/getting-started/first-steps.md +++ b/docs/getting-started/first-steps.md @@ -1,11 +1,11 @@ --- -title: First Steps +title: First steps alias: name: first-steps text: First Steps --- -# First Steps +# First steps !!! Warning Make sure you have read [[getting-started#installation]] before using the library. @@ -36,102 +36,5 @@ by browsing our worked-out examples illustrating pyDVL's capabilities either: ## Advanced usage -Besides the dos and don'ts of data valuation itself, which are the subject of -the examples and the documentation of each method, there are two main things to -keep in mind when using pyDVL. - -### Caching { #getting-started-cache } - -PyDVL can cache (memoize) the computation of the utility function -and speed up some computations for data valuation. -It is however disabled by default. -When it is enabled it takes into account the data indices passed as argument -and the utility function wrapped into the -[Utility][pydvl.utils.utility.Utility] object. This means that -care must be taken when reusing the same utility function with different data, -see the documentation for the [caching package][pydvl.utils.caching] for more -information. - -In general, caching won't play a major role in the computation of Shapley values -because the probability of sampling the same subset twice, and hence needing -the same utility function computation, is very low. However, it can be very -useful when comparing methods that use the same utility function, or when -running multiple experiments with the same data. - -pyDVL supports 3 different caching backends: - -- [InMemoryCacheBackend][pydvl.utils.caching.memory.InMemoryCacheBackend]: - an in-memory cache backend that uses a dictionary to store and retrieve - cached values. This is used to share cached values between threads - in a single process. -- [DiskCacheBackend][pydvl.utils.caching.disk.DiskCacheBackend]: - a disk-based cache backend that uses pickled values written to and read from disk. - This is used to share cached values between processes in a single machine. -- [MemcachedCacheBackend][pydvl.utils.caching.memcached.MemcachedCacheBackend]: - a [Memcached](https://memcached.org/)-based cache backend that uses pickled - values written to and read from a Memcached server. This is used to share - cached values between processes across multiple machines. Note that this - backend requires optional dependencies, see [Extras][installation-extras]. - -!!! tip "When is the cache really necessary?" - Crucially, semi-value computations with the - [PermutationSampler][pydvl.value.sampler.PermutationSampler] require caching - to be enabled, or they will take twice as long as the direct implementation - in [compute_shapley_values][pydvl.value.shapley.compute_shapley_values]. - -!!! tip "Using the cache" - Continue reading about the cache in the documentation - for the [caching package][pydvl.utils.caching]. - -#### Setting up the Memcached cache { #setting-up-memcached } - -[Memcached](https://memcached.org/) is an in-memory key-value store accessible -over the network. pyDVL can use it to cache the computation of the utility function -and speed up some computations (in particular, semi-value computations with the -[PermutationSampler][pydvl.value.sampler.PermutationSampler] but other methods -may benefit as well). - -You can either install it as a package or run it inside a docker container (the -simplest). For installation instructions, refer to the [Getting -started](https://github.com/memcached/memcached/wiki#getting-started) section in -memcached's wiki. Then you can run it with: - -```shell -memcached -u user -``` - -To run memcached inside a container in daemon mode instead, use: - -```shell -docker container run -d --rm -p 11211:11211 memcached:latest -``` - -### Parallelization { #setting-up-parallelization } - -pyDVL uses [joblib](https://joblib.readthedocs.io/en/latest/) for local -parallelization (within one machine) and supports using -[Ray](https://ray.io) for distributed parallelization (across multiple machines). - -The former works out of the box but for the latter you will need to install -additional dependencies (see [Extras][installation-extras] ) -and to provide a running cluster (or run ray in local mode). - -As of v0.8.1 pyDVL does not allow requesting resources per task sent to the -cluster, so you will need to make sure that each worker has enough resources to -handle the tasks it receives. A data valuation task using game-theoretic methods -will typically make a copy of the whole model and dataset to each worker, even -if the re-training only happens on a subset of the data. This means that you -should make sure that each worker has enough memory to handle the whole dataset. - -#### Ray - -Please follow the instructions in Ray's documentation to set up a cluster. -Once you have a running cluster, you can use it by passing the address -of the head node to parallel methods via [ParallelConfig][pydvl.parallel.config.ParallelConfig]. - -For a local ray cluster you would use: - -```python -from pydvl.parallel.config import ParallelConfig -config = ParallelConfig(backend="ray") -``` +Refer to the [[advanced-usage]] page for explanations on how to enable +and use parallelization and caching. diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 9b0c4a7f6..da4b2c1eb 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -1,8 +1,8 @@ --- -title: Getting Started +title: Getting started alias: name: getting-started - title: Getting Started + title: Getting started --- # Getting started @@ -104,4 +104,4 @@ pip install pyDVL[memcached] This installs [pymemcache](https://github.com/pinterest/pymemcache) additionally. Be aware that you still have to start a memcached server manually. -See [Setting up the Memcached cache](first-steps/#setting-up-memcached). +See [Setting up the Memcached cache][setting-up-memcached]. diff --git a/docs/getting-started/methods.md b/docs/getting-started/methods.md index c9dcab2a1..d5f979859 100644 --- a/docs/getting-started/methods.md +++ b/docs/getting-started/methods.md @@ -7,7 +7,7 @@ alias: We currently implement the following methods: -## Data Valuation +## Data valuation - [**LOO**][pydvl.value.loo.compute_loo]. @@ -44,7 +44,7 @@ We currently implement the following methods: - [**Data-OOB**][pydvl.value.oob.compute_data_oob] [@kwon_dataoob_2023]. -## Influence Functions +## Influence functions - [**CG Influence**][pydvl.influence.torch.CgInfluence]. [@koh_understanding_2017]. diff --git a/docs/influence/scaling_computation.md b/docs/influence/scaling_computation.md index f21d8700e..e4f7d7e78 100644 --- a/docs/influence/scaling_computation.md +++ b/docs/influence/scaling_computation.md @@ -1,3 +1,10 @@ +--- +title: Scaling computation +alias: + name: scaling-influence-computation + text: Scaling computation +--- + The implementations of [InfluenceFunctionModel][pydvl.influence.base_influence_function_model.InfluenceFunctionModel] provide a convenient way to calculate influences for in memory tensors. diff --git a/docs/value/index.md b/docs/value/index.md index cac75a0ab..6ddc974d0 100644 --- a/docs/value/index.md +++ b/docs/value/index.md @@ -224,7 +224,7 @@ from sklearn.linear_model import LinearRegression, LogisticRegression from sklearn.datasets import load_iris dataset = Dataset.from_sklearn(load_iris()) -u = Utility(LogisticRegression(), dataset, enable_cache=False) +u = Utility(LogisticRegression(), dataset) training_budget = 3 wrapped_u = DataUtilityLearning(u, training_budget, LinearRegression()) diff --git a/mkdocs.yml b/mkdocs.yml index 8b53292a5..783fb986b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,6 +14,7 @@ nav: - Applications: getting-started/applications.md - Benchmarking: getting-started/benchmarking.md - Methods: getting-started/methods.md + - Advanced usage: getting-started/advanced-usage.md - Glossary: getting-started/glossary.md - Data Valuation: - value/index.md @@ -74,7 +75,6 @@ plugins: canonical_version: stable - section-index - alias: - use_relative_link: true verbose: true - gen-files: scripts: @@ -110,6 +110,7 @@ plugins: - https://joblib.readthedocs.io/en/stable/objects.inv - https://docs.dask.org/en/latest/objects.inv - https://distributed.dask.org/en/latest/objects.inv + - https://docs.ray.io/en/latest/objects.inv paths: [ src ] # search packages in the src folder options: heading_level: 1 diff --git a/requirements.txt b/requirements.txt index aedacc12f..d819d727e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ pandas>=1.3 scikit-learn scipy>=1.7.0 cvxpy>=1.3.0 -joblib +joblib>=1.3.0 cloudpickle tqdm matplotlib diff --git a/src/pydvl/influence/influence_calculator.py b/src/pydvl/influence/influence_calculator.py index 7249ca257..7e1186f29 100644 --- a/src/pydvl/influence/influence_calculator.py +++ b/src/pydvl/influence/influence_calculator.py @@ -155,8 +155,8 @@ class DaskInfluenceCalculator: from pydvl.influence import DaskInfluenceCalculator from pydvl.influence.torch import CgInfluence from pydvl.influence.torch.util import ( - torch_dataset_to_dask_array, - TorchNumpyConverter, + torch_dataset_to_dask_array, + TorchNumpyConverter, ) from distributed import Client diff --git a/src/pydvl/parallel/backends/joblib.py b/src/pydvl/parallel/backends/joblib.py index 48b1dcd77..264fc2a0b 100644 --- a/src/pydvl/parallel/backends/joblib.py +++ b/src/pydvl/parallel/backends/joblib.py @@ -25,6 +25,22 @@ class JoblibParallelBackend(BaseParallelBackend, backend_name="joblib"): It shouldn't be initialized directly. You should instead call [init_parallel_backend()][pydvl.parallel.backend.init_parallel_backend]. + ??? Example + ``` python + from pydvl.parallel import init_paralle_backend, ParallelConfig + config = ParallelConfig(backend="joblib") + parallel_backend = init_parallel_backend(config) + ``` + + ??? Example + ``` python + import joblib + from pydvl.parallel import init_paralle_backend, ParallelConfig + with joblib.parallel_config(verbose=100): + config = ParallelConfig(backend="joblib") + parallel_backend = init_parallel_backend(config) + ``` + Args: config: instance of [ParallelConfig][pydvl.utils.config.ParallelConfig] with cluster address, number of cpus, etc. @@ -32,7 +48,6 @@ class JoblibParallelBackend(BaseParallelBackend, backend_name="joblib"): def __init__(self, config: ParallelConfig): self.config = { - "logging_level": config.logging_level, "n_jobs": config.n_cpus_local, } @@ -70,9 +85,8 @@ def wait(self, v: list[T], *args, **kwargs) -> tuple[list[T], list[T]]: return v, [] def _effective_n_jobs(self, n_jobs: int) -> int: - if self.config["n_jobs"] is None: - maximum_n_jobs = joblib.effective_n_jobs() - else: + eff_n_jobs: int = joblib.effective_n_jobs(n_jobs) + if self.config["n_jobs"] is not None: maximum_n_jobs = self.config["n_jobs"] - eff_n_jobs: int = min(joblib.effective_n_jobs(n_jobs), maximum_n_jobs) + eff_n_jobs = min(eff_n_jobs, maximum_n_jobs) return eff_n_jobs diff --git a/src/pydvl/parallel/backends/ray.py b/src/pydvl/parallel/backends/ray.py index abb33ebe3..3f9cc3f50 100644 --- a/src/pydvl/parallel/backends/ray.py +++ b/src/pydvl/parallel/backends/ray.py @@ -23,20 +23,27 @@ class RayParallelBackend(BaseParallelBackend, backend_name="ray"): It shouldn't be initialized directly. You should instead call [init_parallel_backend()][pydvl.parallel.backend.init_parallel_backend]. + ??? Example + ``` python + import ray + from pydvl.parallel import init_parallel_backend, ParallelConfig + ray.init() + config = ParallelConfig(backend="ray") + parallel_backend = init_parallel_backend(config) + ``` + Args: config: instance of [ParallelConfig][pydvl.utils.config.ParallelConfig] with cluster address, number of cpus, etc. """ def __init__(self, config: ParallelConfig): - self.config = { - "address": config.address, - "logging_level": config.logging_level or logging.WARNING, - } - if self.config["address"] is None: - self.config["num_cpus"] = config.n_cpus_local if not ray.is_initialized(): - ray.init(**self.config) + raise RuntimeError( + "Starting from v0.9.0, ray is no longer automatically initialized. " + "Please use `ray.init()` with the desired configuration " + "before using this class." + ) # Register ray joblib backend register_ray() diff --git a/src/pydvl/parallel/config.py b/src/pydvl/parallel/config.py index 46d2d1ee9..f63921f89 100644 --- a/src/pydvl/parallel/config.py +++ b/src/pydvl/parallel/config.py @@ -1,21 +1,24 @@ -import logging +import warnings from dataclasses import dataclass from typing import Literal, Optional, Tuple, Union __all__ = ["ParallelConfig"] +# TODO: delete this class once it's made redundant in v0.10.0 +# This string for the benefit of deprecation searches: +# remove_in="0.10.0" @dataclass(frozen=True) class ParallelConfig: """Configuration for parallel computation backend. Args: backend: Type of backend to use. Defaults to 'joblib' - address: Address of existing remote or local cluster to use. - n_cpus_local: Number of CPUs to use when creating a local ray cluster. + address: (DEPRECATED) Address of existing remote or local cluster to use. + n_cpus_local: (DEPRECATED) Number of CPUs to use when creating a local ray cluster. This has no effect when using an existing ray cluster. - logging_level: Logging level for the parallel backend's worker. - wait_timeout: Timeout in seconds for waiting on futures. + logging_level: (DEPRECATED) Logging level for the parallel backend's worker. + wait_timeout: (DEPRECATED) Timeout in seconds for waiting on futures. """ backend: Literal["joblib", "ray"] = "joblib" @@ -25,6 +28,30 @@ class ParallelConfig: wait_timeout: float = 1.0 def __post_init__(self) -> None: + warnings.warn( + "The `ParallelConfig` class was deprecated in v0.9.0 and will be removed in v0.10.0", + FutureWarning, + ) + if self.address is not None: + warnings.warn( + "`address` is deprecated in v0.9.0 and will be removed in v0.10.0", + FutureWarning, + ) + if self.n_cpus_local is not None: + warnings.warn( + "`n_cpus_local` is deprecated in v0.9.0 and will be removed in v0.10.0", + FutureWarning, + ) + if self.logging_level is not None: + warnings.warn( + "`logging_level` is deprecated in v0.9.0 and will be removed in v0.10.0", + FutureWarning, + ) + if self.wait_timeout != 1.0: + warnings.warn( + "`wait_timeout` is deprecated in v0.9.0 and will be removed in v0.10.0", + FutureWarning, + ) # FIXME: this is specific to ray if self.address is not None and self.n_cpus_local is not None: raise ValueError("When `address` is set, `n_cpus_local` should be None.") diff --git a/src/pydvl/parallel/futures/__init__.py b/src/pydvl/parallel/futures/__init__.py index c42026ecf..ce42ecc91 100644 --- a/src/pydvl/parallel/futures/__init__.py +++ b/src/pydvl/parallel/futures/__init__.py @@ -7,7 +7,7 @@ try: from pydvl.parallel.futures.ray import RayExecutor -except ImportError: +except ModuleNotFoundError: pass __all__ = ["init_executor"] @@ -30,7 +30,7 @@ def init_executor( ??? Examples ``` python - from pydvl.parallel import init_executor, ParallelConfig + from pydvl.parallel.futures import init_executor, ParallelConfig config = ParallelConfig(backend="ray") with init_executor(max_workers=1, config=config) as executor: diff --git a/src/pydvl/parallel/futures/ray.py b/src/pydvl/parallel/futures/ray.py index b15eef9e0..0aa97f152 100644 --- a/src/pydvl/parallel/futures/ray.py +++ b/src/pydvl/parallel/futures/ray.py @@ -8,14 +8,20 @@ from typing import Any, Callable, Optional, TypeVar from weakref import WeakSet, ref -import ray +try: + import ray +except ModuleNotFoundError as e: + raise ModuleNotFoundError( + f"Cannot use RayExecutor because ray was not installed. " + f"Make sure to install pyDVL using `pip install pyDVL[ray]`. \n" + f"Original error: {e}" + ) +from pydvl.parallel import CancellationPolicy from pydvl.parallel.config import ParallelConfig __all__ = ["RayExecutor"] -from pydvl.parallel import CancellationPolicy - T = TypeVar("T") logger = logging.getLogger(__name__) @@ -68,12 +74,12 @@ def __init__( else CancellationPolicy.NONE ) - self.config = {"address": config.address, "logging_level": config.logging_level} - if config.address is None: - self.config["num_cpus"] = config.n_cpus_local - if not ray.is_initialized(): - ray.init(**self.config) + raise RuntimeError( + "Starting from v0.9.0, ray is no longer automatically initialized. " + "Please use `ray.init()` with the desired configuration " + "before using this class." + ) self._max_workers = max_workers if self._max_workers is None: diff --git a/src/pydvl/parallel/map_reduce.py b/src/pydvl/parallel/map_reduce.py index 401df3fca..a4cfd272a 100644 --- a/src/pydvl/parallel/map_reduce.py +++ b/src/pydvl/parallel/map_reduce.py @@ -124,17 +124,8 @@ def __call__( Returns: The result of the reduce function. """ - parallel_kwargs: Dict[str, Any] = {"n_jobs": self.n_jobs} - if self.config.backend == "joblib": - parallel_kwargs["backend"] = "loky" - else: - parallel_kwargs["backend"] = self.config.backend - # In joblib the levels are reversed. - # 0 means no logging and 50 means log everything to stdout - if self.config.logging_level is not None: - parallel_kwargs["verbose"] = 50 - self.config.logging_level seed_seq = ensure_seed_sequence(seed) - with Parallel(**parallel_kwargs) as parallel: + with Parallel() as parallel: chunks = self._chunkify(self.inputs_, n_chunks=self.n_jobs) map_results: List[R] = parallel( delayed(self._map_func)( diff --git a/src/pydvl/utils/__init__.py b/src/pydvl/utils/__init__.py index 245c596dd..edc27c579 100644 --- a/src/pydvl/utils/__init__.py +++ b/src/pydvl/utils/__init__.py @@ -1,4 +1,3 @@ -from ..parallel import * from .caching import * from .config import * from .dataset import * diff --git a/src/pydvl/utils/caching/disk.py b/src/pydvl/utils/caching/disk.py index 06250a450..c291bbb80 100644 --- a/src/pydvl/utils/caching/disk.py +++ b/src/pydvl/utils/caching/disk.py @@ -26,8 +26,9 @@ class DiskCacheBackend(CacheBackend): Attributes: cache_dir: Base directory for cache storage. - ??? Examples - ``` pycon + Example: + Basic usage: + ```pycon >>> from pydvl.utils.caching.disk import DiskCacheBackend >>> cache_backend = DiskCacheBackend() >>> cache_backend.clear() @@ -37,7 +38,8 @@ class DiskCacheBackend(CacheBackend): 42 ``` - ``` pycon + Callable wrapping: + ```pycon >>> from pydvl.utils.caching.disk import DiskCacheBackend >>> cache_backend = DiskCacheBackend() >>> cache_backend.clear() diff --git a/src/pydvl/utils/caching/memcached.py b/src/pydvl/utils/caching/memcached.py index 1e761cbd4..0eb348f68 100644 --- a/src/pydvl/utils/caching/memcached.py +++ b/src/pydvl/utils/caching/memcached.py @@ -65,8 +65,9 @@ class MemcachedCacheBackend(CacheBackend): config: Memcached client configuration. client: Memcached client instance. - ??? Examples - ``` pycon + Example: + Basic usage: + ```pycon >>> from pydvl.utils.caching.memcached import MemcachedCacheBackend >>> cache_backend = MemcachedCacheBackend() >>> cache_backend.clear() @@ -76,7 +77,8 @@ class MemcachedCacheBackend(CacheBackend): 42 ``` - ``` pycon + Callable wrapping: + ```pycon >>> from pydvl.utils.caching.memcached import MemcachedCacheBackend >>> cache_backend = MemcachedCacheBackend() >>> cache_backend.clear() diff --git a/src/pydvl/utils/caching/memory.py b/src/pydvl/utils/caching/memory.py index 270d3ce1a..b7db38dee 100644 --- a/src/pydvl/utils/caching/memory.py +++ b/src/pydvl/utils/caching/memory.py @@ -18,8 +18,9 @@ class InMemoryCacheBackend(CacheBackend): Attributes: cached_values: Dictionary used to store cached values. - ??? Examples - ``` pycon + Example: + Basic usage: + ```pycon >>> from pydvl.utils.caching.memory import InMemoryCacheBackend >>> cache_backend = InMemoryCacheBackend() >>> cache_backend.clear() @@ -29,7 +30,8 @@ class InMemoryCacheBackend(CacheBackend): 42 ``` - ``` pycon + Callable wrapping: + ```pycon >>> from pydvl.utils.caching.memory import InMemoryCacheBackend >>> cache_backend = InMemoryCacheBackend() >>> cache_backend.clear() diff --git a/src/pydvl/utils/parallel.py b/src/pydvl/utils/parallel.py deleted file mode 100644 index 2df52605f..000000000 --- a/src/pydvl/utils/parallel.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -# This module is deprecated - -!!! warning "Redirects" - Imports from this module will be redirected to - [pydvl.parallel][pydvl.parallel] only until v0.9.0. Please update your - imports. -""" -import logging - -from ..parallel.backend import * -from ..parallel.config import * -from ..parallel.futures import * -from ..parallel.map_reduce import * - -log = logging.getLogger(__name__) - -# This string for the benefit of deprecation searches: -# remove_in="0.9.0" -log.warning( - "Importing parallel tools from pydvl.utils is deprecated. " - "Please import directly from pydvl.parallel. " - "Redirected imports will be removed in v0.9.0" -) diff --git a/tests/parallel/__init__.py b/tests/parallel/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/utils/conftest.py b/tests/parallel/conftest.py similarity index 88% rename from tests/utils/conftest.py rename to tests/parallel/conftest.py index 1bbb7dea0..1326fa7ca 100644 --- a/tests/utils/conftest.py +++ b/tests/parallel/conftest.py @@ -11,7 +11,8 @@ def parallel_config(request): yield ParallelConfig(backend="joblib", n_cpus_local=num_workers()) elif request.param == "ray-local": ray = pytest.importorskip("ray", reason="Ray not installed.") - yield ParallelConfig(backend="ray", n_cpus_local=num_workers()) + ray.init(num_cpus=num_workers()) + yield ParallelConfig(backend="ray") ray.shutdown() elif request.param == "ray-external": ray = pytest.importorskip("ray", reason="Ray not installed.") @@ -22,6 +23,7 @@ def parallel_config(request): cluster = Cluster( initialize_head=True, head_node_args={"num_cpus": num_workers()} ) + ray.init(cluster.address) yield ParallelConfig(backend="ray", address=cluster.address) ray.shutdown() cluster.shutdown() diff --git a/tests/utils/test_parallel.py b/tests/parallel/test_parallel.py similarity index 100% rename from tests/utils/test_parallel.py rename to tests/parallel/test_parallel.py diff --git a/tests/utils/test_caching.py b/tests/utils/test_caching.py index 636fe8ade..846a6dba6 100644 --- a/tests/utils/test_caching.py +++ b/tests/utils/test_caching.py @@ -3,13 +3,13 @@ import tempfile from time import sleep, time from typing import Optional -from unittest.mock import MagicMock, PropertyMock, patch import numpy as np import pytest from numpy.typing import NDArray from pydvl.parallel import MapReduceJob +from pydvl.parallel.config import ParallelConfig from pydvl.utils.caching import ( CachedFunc, CachedFuncConfig, @@ -19,9 +19,16 @@ ) from pydvl.utils.types import Seed +from ..conftest import num_workers + logger = logging.getLogger(__name__) +@pytest.fixture(scope="module") +def parallel_config(): + return ParallelConfig(backend="joblib", n_cpus_local=num_workers()) + + def foo(indices: NDArray[np.int_], *args, **kwargs) -> float: return float(np.sum(indices)) @@ -65,7 +72,12 @@ def cache_backend(request): yield cache_backend cache_backend.clear() elif backend == "memcached": - cache_backend = MemcachedCacheBackend() + try: + cache_backend = MemcachedCacheBackend() + except ConnectionRefusedError as e: + raise RuntimeError( + f"Could not connected to Memcached server. original error message: {str(e)}" + ) yield cache_backend cache_backend.clear() else: @@ -212,12 +224,10 @@ def test_cache_ignore_args(cache_backend): def test_parallel_jobs(cache_backend, parallel_config): if not isinstance(cache_backend, MemcachedCacheBackend): pytest.skip("Only running this test with MemcachedCacheBackend") - if parallel_config.backend != "joblib": - pytest.skip("We don't have to test this with all parallel backends") # Note that we typically do NOT want to ignore run_id cached_func_config = CachedFuncConfig( - ignore_args=["job_id", "run_id"], + ignore_args=["job_id", "run_id"], time_threshold=0 ) wrapped_foo = cache_backend.wrap(foo, config=cached_func_config) @@ -303,9 +313,6 @@ def test_faster_with_repeated_training(cache_backend, worker_id: str): def test_parallel_repeated_training( cache_backend, n, atol, n_jobs, n_runs, parallel_config ): - if parallel_config.backend != "joblib": - pytest.skip("We don't have to test this with all parallel backends") - def map_func(indices: NDArray[np.int_], seed: Optional[Seed] = None) -> float: return np.sum(indices).item() + np.random.normal(scale=1) diff --git a/tests/value/shapley/test_montecarlo.py b/tests/value/shapley/test_montecarlo.py index 58f9df2a9..157ebc40f 100644 --- a/tests/value/shapley/test_montecarlo.py +++ b/tests/value/shapley/test_montecarlo.py @@ -131,10 +131,18 @@ def test_seed( np.testing.assert_equal(values_1.values, values_3.values) +@pytest.mark.skip( + "This test is brittle and the bound isn't sharp. " + "We should at least document the bound in the documentation." +) @pytest.mark.slow @pytest.mark.parametrize("num_samples, delta, eps", [(6, 0.1, 0.1)]) @pytest.mark.parametrize( - "fun", [ShapleyMode.PermutationMontecarlo, ShapleyMode.CombinatorialMontecarlo] + "fun", + [ + ShapleyMode.PermutationMontecarlo, + ShapleyMode.CombinatorialMontecarlo, + ], ) def test_hoeffding_bound_montecarlo( num_samples,